fix: centroid-based pose tracking for responsive limb movement

Rewrites pose decoder from intensity-based to position-based tracking:
- Arms now track toward motion centroid in each body zone
- Elbow/wrist positions computed along shoulder→centroid vector
- Legs track toward lower-body zone centroids
- Smoothing reduced from 0.45 to 0.25 for responsiveness
- Zone centroids blend 30% old / 70% new each frame

6 body zones with overlapping coverage:
- Head (top 20%, center cols)
- Left/Right Arm (rows 10-60%, outer cols)
- Torso (rows 15-55%, center cols)
- Left/Right Leg (rows 50-100%, half cols each)

Hand openness now driven by arm spread distance + raise amount.
Cache busters v=6.

Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
ruv 2026-03-12 19:35:16 -04:00
parent 0ef1252678
commit 24a340bdaf
3 changed files with 173 additions and 151 deletions

View File

@ -4,7 +4,7 @@
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>WiFi-DensePose — Dual-Modal Pose Estimation</title> <title>WiFi-DensePose — Dual-Modal Pose Estimation</title>
<link rel="stylesheet" href="pose-fusion/css/style.css?v=5"> <link rel="stylesheet" href="pose-fusion/css/style.css?v=6">
</head> </head>
<body> <body>
@ -172,6 +172,6 @@
</div><!-- /main-grid --> </div><!-- /main-grid -->
<script type="module" src="pose-fusion/js/main.js?v=5"></script> <script type="module" src="pose-fusion/js/main.js?v=6"></script>
</body> </body>
</html> </html>

View File

@ -4,12 +4,12 @@
* Main orchestration: video capture CNN embedding CSI processing fusion rendering * Main orchestration: video capture CNN embedding CSI processing fusion rendering
*/ */
import { VideoCapture } from './video-capture.js?v=5'; import { VideoCapture } from './video-capture.js?v=6';
import { CsiSimulator } from './csi-simulator.js?v=5'; import { CsiSimulator } from './csi-simulator.js?v=6';
import { CnnEmbedder } from './cnn-embedder.js?v=5'; import { CnnEmbedder } from './cnn-embedder.js?v=6';
import { FusionEngine } from './fusion-engine.js?v=5'; import { FusionEngine } from './fusion-engine.js?v=6';
import { PoseDecoder } from './pose-decoder.js?v=5'; import { PoseDecoder } from './pose-decoder.js?v=6';
import { CanvasRenderer } from './canvas-renderer.js?v=5'; import { CanvasRenderer } from './canvas-renderer.js?v=6';
// === State === // === State ===
let mode = 'dual'; // 'dual' | 'video' | 'csi' let mode = 'dual'; // 'dual' | 'video' | 'csi'

View File

@ -64,7 +64,7 @@ export class PoseDecoder {
constructor(embeddingDim = 128) { constructor(embeddingDim = 128) {
this.embeddingDim = embeddingDim; this.embeddingDim = embeddingDim;
this.smoothedKeypoints = null; this.smoothedKeypoints = null;
this.smoothingFactor = 0.45; // Lower = more responsive to movement this.smoothingFactor = 0.25; // Low = responsive to real movement
this._time = 0; this._time = 0;
// Through-wall tracking state // Through-wall tracking state
@ -73,12 +73,19 @@ export class PoseDecoder {
this._ghostConfidence = 0; this._ghostConfidence = 0;
this._ghostVelocity = { x: 0, y: 0 }; this._ghostVelocity = { x: 0, y: 0 };
// Arm tracking history (smoothed positions) // Zone centroid tracking (normalized 0-1 positions)
this._leftArmY = 0.5; this._headCx = 0.5;
this._rightArmY = 0.5; this._headCy = 0.15;
this._leftArmX = 0; this._leftArmCx = 0.3;
this._rightArmX = 0; this._leftArmCy = 0.35;
this._headOffsetX = 0; this._rightArmCx = 0.7;
this._rightArmCy = 0.35;
this._leftLegCx = 0.4;
this._leftLegCy = 0.8;
this._rightLegCx = 0.6;
this._rightLegCy = 0.8;
this._torsoCx = 0.5;
this._torsoCy = 0.45;
} }
/** /**
@ -142,71 +149,117 @@ export class PoseDecoder {
/** /**
* Track body parts from the motion grid. * Track body parts from the motion grid.
* The grid tells us WHERE motion is happening we map that to joint positions. * Finds the centroid of motion in each body zone and positions joints there.
*/ */
_trackFromMotionGrid(region, embedding, elapsed) { _trackFromMotionGrid(region, embedding, elapsed) {
const grid = region.motionGrid; const grid = region.motionGrid;
const cols = region.gridCols || 10; const cols = region.gridCols || 10;
const rows = region.gridRows || 8; const rows = region.gridRows || 8;
// Body bounding box // Body bounding box (in normalized 0-1 coords)
const cx = region.x + region.w / 2; const bx = region.x, by = region.y, bw = region.w, bh = region.h;
const cy = region.y + region.h / 2; const cx = bx + bw / 2;
const bodyH = Math.max(region.h, 0.3); const cy = by + bh / 2;
const bodyW = Math.max(region.w, 0.15); const bodyH = Math.max(bh, 0.3);
const bodyW = Math.max(bw, 0.15);
// Analyze the motion grid to find arm positions // Find motion centroids per body zone from the grid
// Divide body into zones: head (top 20%), arms (top 60% sides), torso (center), legs (bottom 40%)
if (grid) { if (grid) {
const armAnalysis = this._analyzeArmMotion(grid, cols, rows, region); const zones = this._findZoneCentroids(grid, cols, rows, bx, by, bw, bh);
// Smooth arm tracking // Smooth with low alpha for responsiveness
this._leftArmY = 0.6 * this._leftArmY + 0.4 * armAnalysis.leftArmHeight; const a = 0.3; // 30% old, 70% new → responsive
this._rightArmY = 0.6 * this._rightArmY + 0.4 * armAnalysis.rightArmHeight; this._headCx = a * this._headCx + (1 - a) * zones.head.x;
this._leftArmX = 0.6 * this._leftArmX + 0.4 * armAnalysis.leftArmSpread; this._headCy = a * this._headCy + (1 - a) * zones.head.y;
this._rightArmX = 0.6 * this._rightArmX + 0.4 * armAnalysis.rightArmSpread; this._leftArmCx = a * this._leftArmCx + (1 - a) * zones.leftArm.x;
this._headOffsetX = 0.7 * this._headOffsetX + 0.3 * armAnalysis.headOffsetX; this._leftArmCy = a * this._leftArmCy + (1 - a) * zones.leftArm.y;
this._rightArmCx= a * this._rightArmCx+ (1 - a) * zones.rightArm.x;
this._rightArmCy= a * this._rightArmCy+ (1 - a) * zones.rightArm.y;
this._leftLegCx = a * this._leftLegCx + (1 - a) * zones.leftLeg.x;
this._leftLegCy = a * this._leftLegCy + (1 - a) * zones.leftLeg.y;
this._rightLegCx= a * this._rightLegCx+ (1 - a) * zones.rightLeg.x;
this._rightLegCy= a * this._rightLegCy+ (1 - a) * zones.rightLeg.y;
this._torsoCx = a * this._torsoCx + (1 - a) * zones.torso.x;
this._torsoCy = a * this._torsoCy + (1 - a) * zones.torso.y;
} }
const P = PROPORTIONS; const P = PROPORTIONS;
const halfW = P.shoulderWidth * bodyH / 2;
const hipHalfW = P.hipWidth * bodyH / 2;
// Breathing (subtle) // Breathing (subtle)
const breathe = Math.sin(elapsed * 1.5) * 0.002; const breathe = Math.sin(elapsed * 1.5) * 0.002;
// Core body positions from detection center // === Position joints using tracked centroids ===
const hipY = cy + bodyH * 0.15;
const shoulderY = hipY - P.shoulderToHip * bodyH + breathe;
const headY = shoulderY - P.headToShoulder * bodyH;
const kneeY = hipY + P.hipToKnee * bodyH;
const ankleY = kneeY + P.kneeToAnkle * bodyH;
// HEAD follows motion centroid // HEAD: tracked centroid (top zone)
const headX = cx + this._headOffsetX * bodyW * 0.3; const headX = this._headCx;
const headY = this._headCy;
// ARM POSITIONS driven by motion grid analysis // TORSO center drives shoulder/hip
// leftArmY: 0 = arm down at side, 1 = arm fully raised const torsoX = this._torsoCx;
// leftArmSpread: how far out the arm extends const shoulderY = this._torsoCy - bodyH * 0.08 + breathe;
const leftArmRaise = this._leftArmY; // 0-1 const halfW = P.shoulderWidth * bodyH / 2;
const rightArmRaise = this._rightArmY; const hipHalfW = P.hipWidth * bodyH / 2;
const leftSpread = 0.02 + this._leftArmX * 0.12; const hipY = shoulderY + P.shoulderToHip * bodyH;
const rightSpread = 0.02 + this._rightArmX * 0.12;
// Elbow: interpolate between "at side" and "raised" // ARMS: elbow + wrist driven toward arm zone centroids
const lElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - leftArmRaise * 0.9); // Left arm: shoulder is fixed, elbow/wrist pulled toward left arm centroid
const rElbowY = shoulderY + P.shoulderToElbow * bodyH * (1 - rightArmRaise * 0.9); const lShX = torsoX - halfW;
const lElbowX = cx - halfW - leftSpread; const lShY = shoulderY;
const rElbowX = cx + halfW + rightSpread; // Vector from shoulder toward arm centroid
const lArmDx = this._leftArmCx - lShX;
const lArmDy = this._leftArmCy - lShY;
const lArmDist = Math.sqrt(lArmDx * lArmDx + lArmDy * lArmDy) || 0.01;
const lArmNx = lArmDx / lArmDist;
const lArmNy = lArmDy / lArmDist;
// Elbow at shoulderToElbow distance along that direction
const elbowLen = P.shoulderToElbow * bodyH;
const lElbowX = lShX + lArmNx * elbowLen;
const lElbowY = lShY + lArmNy * elbowLen;
// Wrist continues further
const wristLen = P.elbowToWrist * bodyH;
const lWristX = lElbowX + lArmNx * wristLen;
const lWristY = lElbowY + lArmNy * wristLen;
// Wrist: extends further when raised // Right arm: same approach
const lWristY = lElbowY + P.elbowToWrist * bodyH * (1 - leftArmRaise * 1.1); const rShX = torsoX + halfW;
const rWristY = rElbowY + P.elbowToWrist * bodyH * (1 - rightArmRaise * 1.1); const rShY = shoulderY;
const lWristX = lElbowX - leftSpread * 0.6; const rArmDx = this._rightArmCx - rShX;
const rWristX = rElbowX + rightSpread * 0.6; const rArmDy = this._rightArmCy - rShY;
const rArmDist = Math.sqrt(rArmDx * rArmDx + rArmDy * rArmDy) || 0.01;
const rArmNx = rArmDx / rArmDist;
const rArmNy = rArmDy / rArmDist;
const rElbowX = rShX + rArmNx * elbowLen;
const rElbowY = rShY + rArmNy * elbowLen;
const rWristX = rElbowX + rArmNx * wristLen;
const rWristY = rElbowY + rArmNy * wristLen;
// Leg motion from lower grid cells // LEGS: knees/ankles pulled toward leg zone centroids
const legMotion = grid ? this._analyzeLegMotion(grid, cols, rows) : { left: 0, right: 0 }; const lHipX = torsoX - hipHalfW;
const legSwing = 0.015; const rHipX = torsoX + hipHalfW;
const lLegDx = this._leftLegCx - lHipX;
const lLegDy = Math.max(0.05, this._leftLegCy - hipY); // always downward
const lLegDist = Math.sqrt(lLegDx * lLegDx + lLegDy * lLegDy) || 0.01;
const lLegNx = lLegDx / lLegDist;
const lLegNy = lLegDy / lLegDist;
const kneeLen = P.hipToKnee * bodyH;
const ankleLen = P.kneeToAnkle * bodyH;
const lKneeX = lHipX + lLegNx * kneeLen;
const lKneeY = hipY + lLegNy * kneeLen;
const lAnkleX = lKneeX + lLegNx * ankleLen;
const lAnkleY = lKneeY + lLegNy * ankleLen;
const rLegDx = this._rightLegCx - rHipX;
const rLegDy = Math.max(0.05, this._rightLegCy - hipY);
const rLegDist = Math.sqrt(rLegDx * rLegDx + rLegDy * rLegDy) || 0.01;
const rLegNx = rLegDx / rLegDist;
const rLegNy = rLegDy / rLegDist;
const rKneeX = rHipX + rLegNx * kneeLen;
const rKneeY = hipY + rLegNy * kneeLen;
const rAnkleX = rKneeX + rLegNx * ankleLen;
const rAnkleY = rKneeY + rLegNy * ankleLen;
// Arm raise amount (for hand openness)
const leftArmRaise = Math.max(0, Math.min(1, (shoulderY - this._leftArmCy) / (bodyH * 0.3)));
const rightArmRaise = Math.max(0, Math.min(1, (shoulderY - this._rightArmCy) / (bodyH * 0.3)));
// Compute hand finger positions from wrist-elbow axis // Compute hand finger positions from wrist-elbow axis
const lHandAngle = Math.atan2(lWristY - lElbowY, lWristX - lElbowX); const lHandAngle = Math.atan2(lWristY - lElbowY, lWristX - lElbowX);
@ -214,9 +267,11 @@ export class PoseDecoder {
const fingerLen = P.wristToFinger * bodyH; const fingerLen = P.wristToFinger * bodyH;
const fingerSpr = P.fingerSpread * bodyH; const fingerSpr = P.fingerSpread * bodyH;
// Hand openness driven by motion intensity (more motion = more spread) // Hand openness driven by arm raise + arm lateral spread
const lHandOpen = Math.min(1, leftArmRaise * 0.5 + (this._leftArmX || 0) * 0.5); const lArmSpread = Math.abs(this._leftArmCx - (bx + bw * 0.3)) / (bw * 0.3);
const rHandOpen = Math.min(1, rightArmRaise * 0.5 + (this._rightArmX || 0) * 0.5); const rArmSpread = Math.abs(this._rightArmCx - (bx + bw * 0.7)) / (bw * 0.3);
const lHandOpen = Math.min(1, leftArmRaise * 0.5 + lArmSpread * 0.5);
const rHandOpen = Math.min(1, rightArmRaise * 0.5 + rArmSpread * 0.5);
// Left ankle/knee positions // Left ankle/knee positions
const lAnkleX = cx - hipHalfW + legMotion.left * legSwing * 1.3; const lAnkleX = cx - hipHalfW + legMotion.left * legSwing * 1.3;
@ -240,9 +295,9 @@ export class PoseDecoder {
// 4: right_ear // 4: right_ear
{ x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, { x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 },
// 5: left_shoulder // 5: left_shoulder
{ x: cx - halfW, y: shoulderY, confidence: 0.94 }, { x: lShX, y: lShY, confidence: 0.94 },
// 6: right_shoulder // 6: right_shoulder
{ x: cx + halfW, y: shoulderY, confidence: 0.94 }, { x: rShX, y: rShY, confidence: 0.94 },
// 7: left_elbow // 7: left_elbow
{ x: lElbowX, y: lElbowY, confidence: 0.87 }, { x: lElbowX, y: lElbowY, confidence: 0.87 },
// 8: right_elbow // 8: right_elbow
@ -252,17 +307,17 @@ export class PoseDecoder {
// 10: right_wrist // 10: right_wrist
{ x: rWristX, y: rWristY, confidence: 0.82 }, { x: rWristX, y: rWristY, confidence: 0.82 },
// 11: left_hip // 11: left_hip
{ x: cx - hipHalfW, y: hipY, confidence: 0.91 }, { x: lHipX, y: hipY, confidence: 0.91 },
// 12: right_hip // 12: right_hip
{ x: cx + hipHalfW, y: hipY, confidence: 0.91 }, { x: rHipX, y: hipY, confidence: 0.91 },
// 13: left_knee // 13: left_knee
{ x: lKneeX, y: kneeY, confidence: 0.88 }, { x: lKneeX, y: lKneeY, confidence: 0.88 },
// 14: right_knee // 14: right_knee
{ x: rKneeX, y: kneeY, confidence: 0.88 }, { x: rKneeX, y: rKneeY, confidence: 0.88 },
// 15: left_ankle // 15: left_ankle
{ x: lAnkleX, y: ankleY, confidence: 0.83 }, { x: lAnkleX, y: lAnkleY, confidence: 0.83 },
// 16: right_ankle // 16: right_ankle
{ x: rAnkleX, y: ankleY, confidence: 0.83 }, { x: rAnkleX, y: rAnkleY, confidence: 0.83 },
// === Extended keypoints (17-25) === // === Extended keypoints (17-25) ===
@ -294,15 +349,15 @@ export class PoseDecoder {
// 23: left_foot_index (toe tip) — extends forward from ankle // 23: left_foot_index (toe tip) — extends forward from ankle
{ x: lAnkleX + P.ankleToToe * bodyH * 0.5, { x: lAnkleX + P.ankleToToe * bodyH * 0.5,
y: ankleY + P.ankleToToe * bodyH * 0.3, y: lAnkleY + P.ankleToToe * bodyH * 0.3,
confidence: 0.65 }, confidence: 0.65 },
// 24: right_foot_index // 24: right_foot_index
{ x: rAnkleX + P.ankleToToe * bodyH * 0.5, { x: rAnkleX + P.ankleToToe * bodyH * 0.5,
y: ankleY + P.ankleToToe * bodyH * 0.3, y: rAnkleY + P.ankleToToe * bodyH * 0.3,
confidence: 0.65 }, confidence: 0.65 },
// 25: neck (midpoint between shoulders, slightly above) // 25: neck (midpoint between shoulders, slightly above)
{ x: neckX, y: neckY, confidence: 0.93 }, { x: (lShX + rShX) / 2, y: shoulderY - P.headToShoulder * bodyH * 0.35, confidence: 0.93 },
]; ];
for (let i = 0; i < keypoints.length; i++) { for (let i = 0; i < keypoints.length; i++) {
@ -313,94 +368,61 @@ export class PoseDecoder {
} }
/** /**
* Analyze the motion grid to determine arm positions. * Find weighted motion centroids for each body zone.
* Left side of grid = left side of body, etc. * Divides the bounding box into 6 zones: head, left arm, right arm, torso, left leg, right leg.
* Returns the (x,y) centroid of motion intensity for each zone.
*/ */
_analyzeArmMotion(grid, cols, rows, region) { _findZoneCentroids(grid, cols, rows, bx, by, bw, bh) {
// Body center column // Zone definitions (in grid-relative fractions)
const centerCol = Math.floor(cols / 2); const zones = {
head: { rMin: 0, rMax: 0.2, cMin: 0.25, cMax: 0.75, wx: 0, wy: 0, wt: 0 },
leftArm: { rMin: 0.1, rMax: 0.6, cMin: 0, cMax: 0.35, wx: 0, wy: 0, wt: 0 },
rightArm: { rMin: 0.1, rMax: 0.6, cMin: 0.65, cMax: 1.0, wx: 0, wy: 0, wt: 0 },
torso: { rMin: 0.15, rMax: 0.55, cMin: 0.3, cMax: 0.7, wx: 0, wy: 0, wt: 0 },
leftLeg: { rMin: 0.5, rMax: 1.0, cMin: 0.1, cMax: 0.5, wx: 0, wy: 0, wt: 0 },
rightLeg: { rMin: 0.5, rMax: 1.0, cMin: 0.5, cMax: 0.9, wx: 0, wy: 0, wt: 0 },
};
// Upper body rows (top 60% of detected region) // Accumulate weighted centroids per zone
const upperEnd = Math.floor(rows * 0.6); for (let r = 0; r < rows; r++) {
const ry = r / rows; // 0-1 within grid
for (let c = 0; c < cols; c++) {
const cx_g = c / cols; // 0-1 within grid
const val = grid[r][c];
if (val < 0.005) continue; // skip near-zero motion
// Compute motion intensity for left vs right, at different heights // Map grid position to body-space coordinates (0-1)
let leftUpperMotion = 0, leftMidMotion = 0; const worldX = bx + cx_g * bw;
let rightUpperMotion = 0, rightMidMotion = 0; const worldY = by + ry * bh;
let leftCount = 0, rightCount = 0;
let headMotionX = 0, headMotionWeight = 0;
for (let r = 0; r < upperEnd; r++) { // Assign to matching zones (a cell can contribute to multiple overlapping zones)
const heightWeight = 1.0 - (r / upperEnd) * 0.3; // Upper rows weighted more for (const z of Object.values(zones)) {
if (ry >= z.rMin && ry < z.rMax && cx_g >= z.cMin && cx_g < z.cMax) {
// Head zone: top 25%, center 40% of width z.wx += worldX * val;
if (r < Math.floor(rows * 0.25)) { z.wy += worldY * val;
const headLeft = Math.floor(cols * 0.3); z.wt += val;
const headRight = Math.floor(cols * 0.7); }
for (let c = headLeft; c <= headRight; c++) {
const val = grid[r][c];
headMotionX += (c / cols - 0.5) * val;
headMotionWeight += val;
} }
} }
// Left arm zone: left 40% of grid
for (let c = 0; c < Math.floor(cols * 0.4); c++) {
const val = grid[r][c];
if (r < rows * 0.3) leftUpperMotion += val * heightWeight;
else leftMidMotion += val * heightWeight;
leftCount++;
}
// Right arm zone: right 40% of grid
for (let c = Math.floor(cols * 0.6); c < cols; c++) {
const val = grid[r][c];
if (r < rows * 0.3) rightUpperMotion += val * heightWeight;
else rightMidMotion += val * heightWeight;
rightCount++;
}
} }
// Normalize // Compute centroids with fallback defaults
const leftTotal = leftUpperMotion + leftMidMotion; const centroid = (z, defX, defY) => ({
const rightTotal = rightUpperMotion + rightMidMotion; x: z.wt > 0.01 ? z.wx / z.wt : defX,
const maxMotion = 0.15; // Calibration threshold y: z.wt > 0.01 ? z.wy / z.wt : defY,
weight: z.wt
});
// Arm height: 0 = at side, 1 = raised const midX = bx + bw / 2;
// High motion in upper-left → left arm is raised const midY = by + bh / 2;
const leftArmHeight = Math.min(1, (leftUpperMotion / maxMotion) * 2);
const rightArmHeight = Math.min(1, (rightUpperMotion / maxMotion) * 2);
// Arm spread: how far out from body
const leftArmSpread = Math.min(1, leftTotal / maxMotion);
const rightArmSpread = Math.min(1, rightTotal / maxMotion);
// Head offset
const headOffsetX = headMotionWeight > 0.01 ? headMotionX / headMotionWeight : 0;
return { leftArmHeight, rightArmHeight, leftArmSpread, rightArmSpread, headOffsetX };
}
/**
* Analyze lower grid for leg motion.
*/
_analyzeLegMotion(grid, cols, rows) {
const lowerStart = Math.floor(rows * 0.6);
let leftMotion = 0, rightMotion = 0;
for (let r = lowerStart; r < rows; r++) {
for (let c = 0; c < Math.floor(cols / 2); c++) {
leftMotion += grid[r][c];
}
for (let c = Math.floor(cols / 2); c < cols; c++) {
rightMotion += grid[r][c];
}
}
// Return as -1 to 1 range (asymmetry indicates which leg is moving)
const total = leftMotion + rightMotion + 0.001;
return { return {
left: (leftMotion - rightMotion) / total, head: centroid(zones.head, midX, by + bh * 0.1),
right: (rightMotion - leftMotion) / total leftArm: centroid(zones.leftArm, bx + bw * 0.2, midY - bh * 0.05),
rightArm: centroid(zones.rightArm, bx + bw * 0.8, midY - bh * 0.05),
torso: centroid(zones.torso, midX, midY),
leftLeg: centroid(zones.leftLeg, bx + bw * 0.35,by + bh * 0.75),
rightLeg: centroid(zones.rightLeg, bx + bw * 0.65,by + bh * 0.75),
}; };
} }