From 05f2d2d7cad5a29080d602077be26c12f78cbb77 Mon Sep 17 00:00:00 2001 From: ruv Date: Thu, 12 Mar 2026 21:55:29 -0400 Subject: [PATCH] =?UTF-8?q?deploy:=20rebrand=20DensePose=20=E2=86=92=20RuV?= =?UTF-8?q?iew=20in=20pose-fusion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pose-fusion.html | 8 +- pose-fusion/css/css/style.css | 536 ++++++++++++++++++++++++++ pose-fusion/js/js/canvas-renderer.js | 247 ++++++++++++ pose-fusion/js/js/cnn-embedder.js | 443 +++++++++++++++++++++ pose-fusion/js/js/csi-simulator.js | 363 ++++++++++++++++++ pose-fusion/js/js/fusion-engine.js | 183 +++++++++ pose-fusion/js/js/main.js | 472 +++++++++++++++++++++++ pose-fusion/js/js/pose-decoder.js | 553 +++++++++++++++++++++++++++ pose-fusion/js/js/video-capture.js | 235 ++++++++++++ ui/pose-fusion.html | 8 +- ui/pose-fusion/css/style.css | 2 +- ui/pose-fusion/js/main.js | 2 +- 12 files changed, 3042 insertions(+), 10 deletions(-) create mode 100644 pose-fusion/css/css/style.css create mode 100644 pose-fusion/js/js/canvas-renderer.js create mode 100644 pose-fusion/js/js/cnn-embedder.js create mode 100644 pose-fusion/js/js/csi-simulator.js create mode 100644 pose-fusion/js/js/fusion-engine.js create mode 100644 pose-fusion/js/js/main.js create mode 100644 pose-fusion/js/js/pose-decoder.js create mode 100644 pose-fusion/js/js/video-capture.js diff --git a/pose-fusion.html b/pose-fusion.html index 08ff952a..39cacc23 100644 --- a/pose-fusion.html +++ b/pose-fusion.html @@ -3,7 +3,7 @@ - WiFi-DensePose — Dual-Modal Pose Estimation + RuView — Dual-Modal Pose Estimation @@ -11,7 +11,7 @@
- +
Dual-Modal Pose Estimation — Live Video + WiFi CSI Fusion
@@ -184,11 +184,11 @@
- WiFi-DensePose · Dual-Modal Pose Estimation · + RuView · Dual-Modal Pose Estimation · Architecture: Conv2D → RuVector 6-Stage Attention (Flash+MHA+Hyperbolic+Linear+MoE+L/G) → Fusion → 26-Keypoint Pose
- GitHub · + GitHub · CNN: ruvector-cnn (loading…) · Observatory
diff --git a/pose-fusion/css/css/style.css b/pose-fusion/css/css/style.css new file mode 100644 index 00000000..bdf71704 --- /dev/null +++ b/pose-fusion/css/css/style.css @@ -0,0 +1,536 @@ +/* RuView — Dual-Modal Pose Fusion Demo + Dark theme matching Observatory */ + +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600;700&family=JetBrains+Mono:wght@400;600&display=swap'); + +:root { + --bg-deep: #080c14; + --bg-panel: rgba(8, 16, 28, 0.92); + --bg-panel-border: rgba(0, 210, 120, 0.25); + --green-glow: #00d878; + --green-bright:#3eff8a; + --green-dim: #0a6b3a; + --amber: #ffb020; + --amber-dim: #a06800; + --blue-signal: #2090ff; + --blue-dim: #0a3060; + --red-alert: #ff3040; + --cyan: #00e5ff; + --text-primary: #e8ece0; + --text-secondary: rgba(232,236,224, 0.55); + --text-label: rgba(232,236,224, 0.35); + --radius: 8px; +} + +* { margin: 0; padding: 0; box-sizing: border-box; } + +body { + background: var(--bg-deep); + font-family: 'Inter', -apple-system, sans-serif; + color: var(--text-primary); + -webkit-font-smoothing: antialiased; + overflow-x: hidden; + min-height: 100vh; +} + +/* === Header === */ +.header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 16px 24px; + border-bottom: 1px solid var(--bg-panel-border); + background: var(--bg-panel); + backdrop-filter: blur(12px); +} + +.header-left { + display: flex; + align-items: center; + gap: 16px; +} + +.logo { + font-weight: 700; + font-size: 24px; + color: var(--green-glow); +} + +.logo .pi { font-style: normal; } + +.header-title { + font-size: 14px; + color: var(--text-secondary); + font-weight: 300; +} + +.header-right { + display: flex; + align-items: center; + gap: 16px; +} + +.mode-select { + background: rgba(0,210,120,0.1); + border: 1px solid var(--bg-panel-border); + color: var(--text-primary); + padding: 6px 12px; + border-radius: var(--radius); + font-family: inherit; + font-size: 13px; + cursor: pointer; +} + +.mode-select option { background: #0c1420; } + +.status-badge { + display: flex; + align-items: center; + gap: 6px; + font-family: 'JetBrains Mono', monospace; + font-size: 12px; + padding: 4px 10px; + border-radius: 12px; + background: rgba(0,210,120,0.1); + border: 1px solid var(--bg-panel-border); +} + +.status-dot { + width: 8px; height: 8px; + border-radius: 50%; + background: var(--green-glow); + box-shadow: 0 0 8px var(--green-glow); + animation: pulse-dot 2s ease infinite; +} + +.status-dot.offline { background: #555; box-shadow: none; animation: none; } +.status-dot.warning { background: var(--amber); box-shadow: 0 0 8px var(--amber); } + +@keyframes pulse-dot { + 0%, 100% { opacity: 1; } + 50% { opacity: 0.5; } +} + +.fps-badge { + font-family: 'JetBrains Mono', monospace; + font-size: 12px; + color: var(--green-glow); +} + +.back-link { + color: var(--text-secondary); + text-decoration: none; + font-size: 13px; + transition: color 0.2s; +} +.back-link:hover { color: var(--green-glow); } + +/* === Main Layout === */ +.main-grid { + display: grid; + grid-template-columns: 1fr 360px; + grid-template-rows: 1fr auto; + gap: 16px; + padding: 16px 24px; + height: calc(100vh - 72px); + overflow: hidden; +} + +.video-panel { + grid-row: 1; +} + +.side-panels { + grid-row: 1; +} + +/* === Video Panel === */ +.video-panel { + position: relative; + background: #000; + border-radius: var(--radius); + border: 1px solid var(--bg-panel-border); + overflow: hidden; + min-height: 0; +} + +.video-panel video { + width: 100%; + height: 100%; + object-fit: cover; + transform: scaleX(-1); +} + +.video-panel canvas { + position: absolute; + top: 0; left: 0; + width: 100%; + height: 100%; + transform: scaleX(-1); +} + +.video-overlay-label { + position: absolute; + top: 12px; left: 12px; + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + padding: 4px 8px; + background: rgba(0,0,0,0.7); + border-radius: 4px; + color: var(--green-glow); + z-index: 5; + transform: scaleX(-1); +} + +.camera-prompt { + position: absolute; + top: 0; left: 0; right: 0; bottom: 0; + display: flex; + flex-direction: column; + align-items: center; + justify-content: center; + text-align: center; + color: var(--text-secondary); + padding: 24px; + z-index: 6; + background: radial-gradient(ellipse at center, rgba(0,210,120,0.08) 0%, rgba(8,12,20,0.95) 70%); +} + +.camera-prompt button { + margin-top: 16px; + padding: 10px 24px; + background: var(--green-glow); + color: #000; + border: none; + border-radius: var(--radius); + font-family: inherit; + font-weight: 600; + font-size: 14px; + cursor: pointer; + transition: background 0.2s; +} + +.camera-prompt button:hover { background: var(--green-bright); } + +.camera-prompt-label { + font-family: 'JetBrains Mono', monospace; + font-size: 14px; + font-weight: 600; + letter-spacing: 2px; + color: var(--green-glow); + text-shadow: 0 0 12px rgba(0,216,120,0.4); + margin-bottom: 12px; +} + +/* === Side Panels === */ +.side-panels { + display: flex; + flex-direction: column; + gap: 8px; + overflow-y: auto; + min-height: 0; + max-height: 100%; + scrollbar-width: thin; + scrollbar-color: var(--green-dim) transparent; +} + +.panel { + background: var(--bg-panel); + border: 1px solid var(--bg-panel-border); + border-radius: var(--radius); + padding: 10px 14px; + flex-shrink: 0; +} + +.panel-title { + font-size: 11px; + text-transform: uppercase; + letter-spacing: 1.2px; + color: var(--text-label); + margin-bottom: 10px; + display: flex; + align-items: center; + gap: 6px; +} + +/* === CSI Heatmap === */ +.csi-canvas-wrapper { + position: relative; + border-radius: 4px; + overflow: hidden; + background: #000; +} + +.csi-canvas-wrapper canvas { + width: 100%; + display: block; +} + +/* === Fusion Bars === */ +.fusion-bars { + display: flex; + flex-direction: column; + gap: 8px; +} + +.bar-row { + display: flex; + align-items: center; + gap: 8px; +} + +.bar-label { + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + color: var(--text-secondary); + width: 55px; + text-align: right; +} + +.bar-track { + flex: 1; + height: 6px; + background: rgba(255,255,255,0.06); + border-radius: 3px; + overflow: hidden; +} + +.bar-fill { + height: 100%; + border-radius: 3px; + transition: width 0.3s ease; +} + +.bar-fill.video { background: var(--cyan); } +.bar-fill.csi { background: var(--amber); } +.bar-fill.fused { background: var(--green-glow); box-shadow: 0 0 8px var(--green-glow); } + +.bar-value { + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + color: var(--text-primary); + width: 36px; +} + +/* === Embedding Space === */ +.embedding-canvas-wrapper { + position: relative; + background: #000; + border-radius: 4px; + overflow: hidden; +} +.embedding-canvas-wrapper canvas { + width: 100%; + display: block; +} + +/* === RuVector Pipeline === */ +.rv-pipeline { + display: flex; + align-items: center; + gap: 2px; + margin-bottom: 8px; + flex-wrap: wrap; +} + +.rv-stage { + font-family: 'JetBrains Mono', monospace; + font-size: 10px; + padding: 3px 6px; + border-radius: 3px; + background: rgba(0,210,120,0.12); + border: 1px solid rgba(0,210,120,0.3); + color: var(--green-glow); + transition: all 0.3s; +} + +.rv-stage.active { + background: rgba(0,210,120,0.25); + box-shadow: 0 0 6px rgba(0,210,120,0.3); +} + +.rv-arrow { + font-size: 10px; + color: var(--text-label); +} + +.rv-stats { + display: flex; + gap: 12px; + font-family: 'JetBrains Mono', monospace; + font-size: 10px; + color: var(--text-secondary); +} + +/* === Latency Panel === */ +.latency-grid { + display: grid; + grid-template-columns: repeat(4, 1fr); + gap: 6px; +} + +.latency-item { + text-align: center; + padding: 6px 0; +} + +.latency-value { + font-family: 'JetBrains Mono', monospace; + font-size: 16px; + font-weight: 600; + color: var(--green-glow); +} + +.latency-label { + font-size: 10px; + color: var(--text-label); + margin-top: 2px; +} + +/* === Controls === */ +.controls-row { + display: flex; + gap: 8px; + flex-wrap: wrap; +} + +.btn { + padding: 6px 14px; + border: 1px solid var(--bg-panel-border); + background: rgba(0,210,120,0.08); + color: var(--text-primary); + border-radius: var(--radius); + font-family: inherit; + font-size: 12px; + cursor: pointer; + transition: all 0.2s; +} +.btn:hover { background: rgba(0,210,120,0.2); } +.btn.active { background: var(--green-glow); color: #000; font-weight: 600; } + +.slider-row { + display: flex; + align-items: center; + gap: 8px; + margin-top: 8px; +} + +.slider-row label { + font-size: 11px; + color: var(--text-secondary); + white-space: nowrap; +} + +.slider-row input[type=range] { + flex: 1; + accent-color: var(--green-glow); +} + +.slider-row .slider-val { + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + width: 32px; + color: var(--green-glow); +} + +/* === Bottom Bar === */ +.bottom-bar { + grid-column: 1 / -1; + display: flex; + align-items: center; + justify-content: space-between; + padding: 10px 16px; + background: var(--bg-panel); + border: 1px solid var(--bg-panel-border); + border-radius: var(--radius); + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + color: var(--text-secondary); +} + +.bottom-bar a { + color: var(--green-glow); + text-decoration: none; +} + +/* === RSSI Signal Strength === */ +.rssi-row { + display: flex; + align-items: center; + gap: 12px; +} + +.rssi-gauge { flex: 1; } + +.rssi-bar-track { + height: 8px; + background: rgba(255,255,255,0.06); + border-radius: 4px; + overflow: hidden; + position: relative; +} + +.rssi-bar-fill { + height: 100%; + border-radius: 4px; + background: linear-gradient(90deg, var(--red-alert), var(--amber), var(--green-glow)); + transition: width 0.4s ease; + position: relative; + box-shadow: 0 0 6px rgba(0,210,120,0.3); +} + +.rssi-bar-fill::after { + content: ''; + position: absolute; + top: 0; left: 0; right: 0; bottom: 0; + background: linear-gradient(90deg, transparent 0%, rgba(255,255,255,0.2) 50%, transparent 100%); + animation: rssi-shimmer 2s ease-in-out infinite; +} + +@keyframes rssi-shimmer { + 0% { transform: translateX(-100%); } + 100% { transform: translateX(100%); } +} + +.rssi-values { + display: flex; + justify-content: space-between; + margin-top: 4px; +} + +.rssi-dbm { + font-family: 'JetBrains Mono', monospace; + font-size: 14px; + font-weight: 600; + color: var(--green-glow); +} + +.rssi-quality { + font-family: 'JetBrains Mono', monospace; + font-size: 11px; + color: var(--text-secondary); + text-transform: uppercase; +} + +#rssi-sparkline { + flex-shrink: 0; + border-radius: 4px; + background: rgba(0,0,0,0.3); +} + +/* === Skeleton colors === */ +.skeleton-joint { fill: var(--green-glow); } +.skeleton-limb { stroke: var(--green-bright); } +.skeleton-joint-csi { fill: var(--amber); } +.skeleton-limb-csi { stroke: var(--amber); } + +/* === Responsive === */ +@media (max-width: 900px) { + .main-grid { + grid-template-columns: 1fr; + height: auto; + overflow: auto; + } + .video-panel { aspect-ratio: 16/9; max-height: 50vh; } + .side-panels { max-height: none; overflow: visible; } +} diff --git a/pose-fusion/js/js/canvas-renderer.js b/pose-fusion/js/js/canvas-renderer.js new file mode 100644 index 00000000..8ac169d9 --- /dev/null +++ b/pose-fusion/js/js/canvas-renderer.js @@ -0,0 +1,247 @@ +/** + * CanvasRenderer — Renders skeleton overlay on video, CSI heatmap, + * embedding space visualization, and fusion confidence bars. + */ + +import { SKELETON_CONNECTIONS } from './pose-decoder.js'; + +export class CanvasRenderer { + constructor() { + this.colors = { + joint: '#00d878', + jointGlow: 'rgba(0, 216, 120, 0.4)', + limb: '#3eff8a', + limbGlow: 'rgba(62, 255, 138, 0.15)', + csiJoint: '#ffb020', + csiLimb: '#ffc850', + fused: '#00e5ff', + confidence: 'rgba(255,255,255,0.3)', + videoEmb: '#00e5ff', + csiEmb: '#ffb020', + fusedEmb: '#00d878', + }; + } + + /** + * Draw skeleton overlay on the video canvas + * @param {CanvasRenderingContext2D} ctx + * @param {Array<{x,y,confidence}>} keypoints - Normalized [0,1] coordinates + * @param {number} width - Canvas width + * @param {number} height - Canvas height + * @param {object} opts + */ + drawSkeleton(ctx, keypoints, width, height, opts = {}) { + const minConf = opts.minConfidence || 0.3; + const color = opts.color || 'green'; + const jointColor = color === 'amber' ? this.colors.csiJoint : this.colors.joint; + const limbColor = color === 'amber' ? this.colors.csiLimb : this.colors.limb; + const glowColor = color === 'amber' ? 'rgba(255,176,32,0.4)' : this.colors.jointGlow; + + ctx.clearRect(0, 0, width, height); + + if (!keypoints || keypoints.length === 0) return; + + // Draw limbs first (behind joints) + ctx.lineWidth = 3; + ctx.lineCap = 'round'; + + for (const [i, j] of SKELETON_CONNECTIONS) { + const kpA = keypoints[i]; + const kpB = keypoints[j]; + if (!kpA || !kpB || kpA.confidence < minConf || kpB.confidence < minConf) continue; + + const ax = kpA.x * width, ay = kpA.y * height; + const bx = kpB.x * width, by = kpB.y * height; + const avgConf = (kpA.confidence + kpB.confidence) / 2; + + // Glow + ctx.strokeStyle = this.colors.limbGlow; + ctx.lineWidth = 8; + ctx.globalAlpha = avgConf * 0.4; + ctx.beginPath(); + ctx.moveTo(ax, ay); + ctx.lineTo(bx, by); + ctx.stroke(); + + // Main line + ctx.strokeStyle = limbColor; + ctx.lineWidth = 2.5; + ctx.globalAlpha = avgConf; + ctx.beginPath(); + ctx.moveTo(ax, ay); + ctx.lineTo(bx, by); + ctx.stroke(); + } + + // Draw joints + ctx.globalAlpha = 1; + for (const kp of keypoints) { + if (!kp || kp.confidence < minConf) continue; + + const x = kp.x * width; + const y = kp.y * height; + const r = 3 + kp.confidence * 3; + + // Glow + ctx.beginPath(); + ctx.arc(x, y, r + 4, 0, Math.PI * 2); + ctx.fillStyle = glowColor; + ctx.globalAlpha = kp.confidence * 0.6; + ctx.fill(); + + // Joint dot + ctx.beginPath(); + ctx.arc(x, y, r, 0, Math.PI * 2); + ctx.fillStyle = jointColor; + ctx.globalAlpha = kp.confidence; + ctx.fill(); + + // White center + ctx.beginPath(); + ctx.arc(x, y, r * 0.4, 0, Math.PI * 2); + ctx.fillStyle = '#fff'; + ctx.globalAlpha = kp.confidence * 0.8; + ctx.fill(); + } + + ctx.globalAlpha = 1; + + // Confidence label + if (opts.label) { + ctx.font = '11px "JetBrains Mono", monospace'; + ctx.fillStyle = jointColor; + ctx.globalAlpha = 0.8; + ctx.fillText(opts.label, 8, height - 8); + ctx.globalAlpha = 1; + } + } + + /** + * Draw CSI amplitude heatmap + * @param {CanvasRenderingContext2D} ctx + * @param {{ data: Float32Array, width: number, height: number }} heatmap + * @param {number} canvasW + * @param {number} canvasH + */ + drawCsiHeatmap(ctx, heatmap, canvasW, canvasH) { + ctx.clearRect(0, 0, canvasW, canvasH); + + if (!heatmap || !heatmap.data || heatmap.height < 2) { + ctx.fillStyle = '#0a0e18'; + ctx.fillRect(0, 0, canvasW, canvasH); + ctx.font = '11px "JetBrains Mono", monospace'; + ctx.fillStyle = 'rgba(255,255,255,0.3)'; + ctx.fillText('Waiting for CSI data...', 8, canvasH / 2); + return; + } + + const { data, width: dw, height: dh } = heatmap; + const cellW = canvasW / dw; + const cellH = canvasH / dh; + + for (let y = 0; y < dh; y++) { + for (let x = 0; x < dw; x++) { + const val = Math.min(1, Math.max(0, data[y * dw + x])); + ctx.fillStyle = this._heatmapColor(val); + ctx.fillRect(x * cellW, y * cellH, cellW + 0.5, cellH + 0.5); + } + } + + // Axis labels + ctx.font = '9px "JetBrains Mono", monospace'; + ctx.fillStyle = 'rgba(255,255,255,0.4)'; + ctx.fillText('Subcarrier →', 4, canvasH - 4); + ctx.save(); + ctx.translate(canvasW - 4, canvasH - 4); + ctx.rotate(-Math.PI / 2); + ctx.fillText('Time ↑', 0, 0); + ctx.restore(); + } + + /** + * Draw embedding space 2D projection + * @param {CanvasRenderingContext2D} ctx + * @param {{ video: Array, csi: Array, fused: Array }} points + * @param {number} w + * @param {number} h + */ + drawEmbeddingSpace(ctx, points, w, h) { + ctx.fillStyle = '#050810'; + ctx.fillRect(0, 0, w, h); + + // Grid + ctx.strokeStyle = 'rgba(255,255,255,0.05)'; + ctx.lineWidth = 0.5; + for (let i = 0; i <= 4; i++) { + const x = (i / 4) * w; + ctx.beginPath(); ctx.moveTo(x, 0); ctx.lineTo(x, h); ctx.stroke(); + const y = (i / 4) * h; + ctx.beginPath(); ctx.moveTo(0, y); ctx.lineTo(w, y); ctx.stroke(); + } + + // Axes + ctx.strokeStyle = 'rgba(255,255,255,0.1)'; + ctx.lineWidth = 1; + ctx.beginPath(); ctx.moveTo(w / 2, 0); ctx.lineTo(w / 2, h); ctx.stroke(); + ctx.beginPath(); ctx.moveTo(0, h / 2); ctx.lineTo(w, h / 2); ctx.stroke(); + + const drawPoints = (pts, color, size) => { + if (!pts || pts.length === 0) return; + const len = pts.length; + for (let i = 0; i < len; i++) { + const p = pts[i]; + if (!p) continue; + const age = 1 - (i / len) * 0.7; // Fade older points + const px = w / 2 + p[0] * w * 0.35; + const py = h / 2 + p[1] * h * 0.35; + + if (px < 0 || px > w || py < 0 || py > h) continue; + + ctx.beginPath(); + ctx.arc(px, py, size, 0, Math.PI * 2); + ctx.fillStyle = color; + ctx.globalAlpha = age * 0.7; + ctx.fill(); + } + }; + + drawPoints(points.video, this.colors.videoEmb, 3); + drawPoints(points.csi, this.colors.csiEmb, 3); + drawPoints(points.fused, this.colors.fusedEmb, 4); + ctx.globalAlpha = 1; + + // Legend + ctx.font = '9px "JetBrains Mono", monospace'; + const legends = [ + { color: this.colors.videoEmb, label: 'Video' }, + { color: this.colors.csiEmb, label: 'CSI' }, + { color: this.colors.fusedEmb, label: 'Fused' }, + ]; + legends.forEach((l, i) => { + const ly = 12 + i * 14; + ctx.fillStyle = l.color; + ctx.beginPath(); + ctx.arc(10, ly - 3, 3, 0, Math.PI * 2); + ctx.fill(); + ctx.fillStyle = 'rgba(255,255,255,0.5)'; + ctx.fillText(l.label, 18, ly); + }); + } + + _heatmapColor(val) { + // Dark blue → cyan → green → yellow → red + if (val < 0.25) { + const t = val / 0.25; + return `rgb(${Math.floor(t * 20)}, ${Math.floor(20 + t * 60)}, ${Math.floor(60 + t * 100)})`; + } else if (val < 0.5) { + const t = (val - 0.25) / 0.25; + return `rgb(${Math.floor(20 + t * 20)}, ${Math.floor(80 + t * 100)}, ${Math.floor(160 - t * 60)})`; + } else if (val < 0.75) { + const t = (val - 0.5) / 0.25; + return `rgb(${Math.floor(40 + t * 180)}, ${Math.floor(180 + t * 75)}, ${Math.floor(100 - t * 80)})`; + } else { + const t = (val - 0.75) / 0.25; + return `rgb(${Math.floor(220 + t * 35)}, ${Math.floor(255 - t * 120)}, ${Math.floor(20 - t * 20)})`; + } + } +} diff --git a/pose-fusion/js/js/cnn-embedder.js b/pose-fusion/js/js/cnn-embedder.js new file mode 100644 index 00000000..10039319 --- /dev/null +++ b/pose-fusion/js/js/cnn-embedder.js @@ -0,0 +1,443 @@ +/** + * CNN Embedder — RuVector Attention-powered feature extractor. + * + * Uses the real ruvector-attention-wasm WASM module for Multi-Head Attention + * and Flash Attention on CSI/video data. Falls back to a JS Conv2D pipeline + * when WASM is not available. + * + * Pipeline: Conv2D → BatchNorm → ReLU → Pool → RuVector Attention → Project → L2 Normalize + * Two instances are created: one for video frames, one for CSI pseudo-images. + */ + +// Seeded PRNG for deterministic weight initialization +function mulberry32(seed) { + return function() { + let t = (seed += 0x6D2B79F5); + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +export class CnnEmbedder { + /** + * @param {object} opts + * @param {number} opts.inputSize - Square input dimension (default 56 for speed) + * @param {number} opts.embeddingDim - Output embedding dimension (default 128) + * @param {boolean} opts.normalize - L2 normalize output + * @param {number} opts.seed - PRNG seed for weight init + */ + constructor(opts = {}) { + this.inputSize = opts.inputSize || 56; + this.embeddingDim = opts.embeddingDim || 128; + this.normalize = opts.normalize !== false; + this.wasmEmbedder = null; + this.rvAttention = null; // RuVector Multi-Head Attention (WASM) + this.rvFlash = null; // RuVector Flash Attention (WASM) + this.rvHyperbolic = null; // RuVector Hyperbolic Attention (hierarchical body) + this.rvMoE = null; // RuVector Mixture-of-Experts (body-region routing) + this.rvLinear = null; // RuVector Linear Attention (O(n) fast hand refinement) + this.rvLocalGlobal = null; // RuVector Local-Global Attention (detail + context) + this.rvModule = null; // RuVector WASM module reference + this.useRuVector = false; + + // Initialize weights with deterministic PRNG + const rng = mulberry32(opts.seed || 42); + const randRange = (lo, hi) => lo + rng() * (hi - lo); + + // Conv 3x3: 3 input channels → 16 output channels + this.convWeights = new Float32Array(3 * 3 * 3 * 16); + for (let i = 0; i < this.convWeights.length; i++) { + this.convWeights[i] = randRange(-0.15, 0.15); + } + + // BatchNorm params (16 channels) + this.bnGamma = new Float32Array(16).fill(1.0); + this.bnBeta = new Float32Array(16).fill(0.0); + this.bnMean = new Float32Array(16).fill(0.0); + this.bnVar = new Float32Array(16).fill(1.0); + + // Projection: 16 → embeddingDim (used when RuVector not available) + this.projWeights = new Float32Array(16 * this.embeddingDim); + for (let i = 0; i < this.projWeights.length; i++) { + this.projWeights[i] = randRange(-0.1, 0.1); + } + + // Attention projection: attention_dim → embeddingDim + this.attnProjWeights = new Float32Array(16 * this.embeddingDim); + for (let i = 0; i < this.attnProjWeights.length; i++) { + this.attnProjWeights[i] = randRange(-0.08, 0.08); + } + } + + /** + * Try to load RuVector attention WASM, then fall back to ruvector-cnn-wasm + * @param {string} wasmPath - Path to the WASM package directory + */ + async tryLoadWasm(wasmPath) { + // First try: RuVector Attention WASM (the real thing — browser ESM build) + try { + const attnBase = new URL('../pkg/ruvector-attention/ruvector_attention_browser.js', import.meta.url).href; + const mod = await import(attnBase); + await mod.default(); // async WASM init via fetch + mod.init(); + + // Create all 6 attention mechanisms + this.rvAttention = new mod.WasmMultiHeadAttention(16, 4); + this.rvFlash = new mod.WasmFlashAttention(16, 8); + this.rvHyperbolic = new mod.WasmHyperbolicAttention(16, -1.0); + this.rvMoE = new mod.WasmMoEAttention(16, 3, 2); + this.rvLinear = new mod.WasmLinearAttention(16, 16); + this.rvLocalGlobal = new mod.WasmLocalGlobalAttention(16, 4, 2); + this.rvModule = mod; + this.useRuVector = true; + + // Log available mechanisms + const mechs = mod.available_mechanisms(); + console.log(`[CNN] RuVector WASM v${mod.version()} — all 6 attention mechanisms active`, mechs); + return true; + } catch (e) { + console.log('[CNN] RuVector Attention WASM not available:', e.message); + } + + // Second try: ruvector-cnn-wasm (legacy path) + try { + const mod = await import(`${wasmPath}/ruvector_cnn_wasm.js`); + await mod.default(); + const config = new mod.EmbedderConfig(); + config.input_size = this.inputSize; + config.embedding_dim = this.embeddingDim; + config.normalize = this.normalize; + this.wasmEmbedder = new mod.WasmCnnEmbedder(config); + console.log('[CNN] WASM CNN embedder loaded successfully'); + return true; + } catch (e) { + console.log('[CNN] WASM CNN not available, using JS fallback:', e.message); + return false; + } + } + + /** + * Extract embedding from RGB image data + * @param {Uint8Array} rgbData - RGB pixel data (H*W*3) + * @param {number} width + * @param {number} height + * @returns {Float32Array} embedding vector + */ + extract(rgbData, width, height) { + if (this.wasmEmbedder) { + try { + const result = this.wasmEmbedder.extract(rgbData, width, height); + return new Float32Array(result); + } catch (_) { /* fallback to JS */ } + } + return this._extractJS(rgbData, width, height); + } + + _extractJS(rgbData, width, height) { + // 1. Resize to inputSize × inputSize if needed + const sz = this.inputSize; + let input; + if (width === sz && height === sz) { + input = new Float32Array(rgbData.length); + for (let i = 0; i < rgbData.length; i++) input[i] = rgbData[i] / 255.0; + } else { + input = this._resize(rgbData, width, height, sz, sz); + } + + // 2. ImageNet normalization + const mean = [0.485, 0.456, 0.406]; + const std = [0.229, 0.224, 0.225]; + const pixels = sz * sz; + for (let i = 0; i < pixels; i++) { + input[i * 3] = (input[i * 3] - mean[0]) / std[0]; + input[i * 3 + 1] = (input[i * 3 + 1] - mean[1]) / std[1]; + input[i * 3 + 2] = (input[i * 3 + 2] - mean[2]) / std[2]; + } + + // 3. Conv2D 3x3 (3 → 16 channels) + const convOut = this._conv2d3x3(input, sz, sz, 3, 16); + + // 4. BatchNorm + this._batchNorm(convOut, 16); + + // 5. ReLU + for (let i = 0; i < convOut.length; i++) { + if (convOut[i] < 0) convOut[i] = 0; + } + + // 6. Global average pooling → spatial tokens (each 16-dim) + const outH = sz - 2, outW = sz - 2; + const spatial = outH * outW; + + // 7. RuVector Attention (if loaded) — apply attention over spatial tokens + if (this.useRuVector && this.rvAttention) { + return this._extractWithAttention(convOut, spatial, 16); + } + + // Fallback: simple global average pool + linear projection + const pooled = new Float32Array(16); + for (let i = 0; i < spatial; i++) { + for (let c = 0; c < 16; c++) { + pooled[c] += convOut[i * 16 + c]; + } + } + for (let c = 0; c < 16; c++) pooled[c] /= spatial; + + // Linear projection → embeddingDim + const emb = new Float32Array(this.embeddingDim); + for (let o = 0; o < this.embeddingDim; o++) { + let sum = 0; + for (let i = 0; i < 16; i++) { + sum += pooled[i] * this.projWeights[i * this.embeddingDim + o]; + } + emb[o] = sum; + } + + // L2 normalize + if (this.normalize) { + let norm = 0; + for (let i = 0; i < emb.length; i++) norm += emb[i] * emb[i]; + norm = Math.sqrt(norm); + if (norm > 1e-8) { + for (let i = 0; i < emb.length; i++) emb[i] /= norm; + } + } + + return emb; + } + + /** + * Full 6-stage RuVector WASM attention pipeline: + * 1. Flash Attention (efficient O(n) pre-screening of spatial tokens) + * 2. Multi-Head Attention (global spatial reasoning) + * 3. Hyperbolic Attention (hierarchical body-part structure, Poincaré ball) + * 4. Linear Attention (O(n) refinement for fine detail — hands/extremities) + * 5. MoE Attention (body-region specialized expert routing) + * 6. Local-Global Attention (local detail + global context fusion) + * → Weighted blend + batch_normalize + project + L2 normalize + */ + _extractWithAttention(convOut, numTokens, channels) { + const mod = this.rvModule; + + // Subsample spatial tokens for attention (max 64 for speed) + const maxTokens = 64; + const step = numTokens > maxTokens ? Math.floor(numTokens / maxTokens) : 1; + const tokens = []; + for (let i = 0; i < numTokens && tokens.length < maxTokens; i += step) { + const token = new Float32Array(channels); + for (let c = 0; c < channels; c++) { + token[c] = convOut[i * channels + c]; + } + tokens.push(token); + } + + const numQueries = Math.min(4, tokens.length); + const queryStride = Math.floor(tokens.length / numQueries); + + // === Stage 1: Flash Attention (efficient pre-screening) === + const flashOut = new Float32Array(channels); + try { + // Flash attention with block size 8 for efficient O(n) screening + const result = this.rvFlash.compute(tokens[0], tokens, tokens); + for (let c = 0; c < channels; c++) flashOut[c] = result[c]; + } catch (_) { + flashOut.set(tokens[0]); + } + + // === Stage 2: Multi-Head Attention (global spatial reasoning) === + const mhaOut = new Float32Array(channels); + for (let q = 0; q < numQueries; q++) { + const queryToken = tokens[q * queryStride]; + try { + const result = this.rvAttention.compute(queryToken, tokens, tokens); + for (let c = 0; c < channels; c++) mhaOut[c] += result[c] / numQueries; + } catch (_) { + for (let c = 0; c < channels; c++) mhaOut[c] += queryToken[c] / numQueries; + } + } + + // === Stage 3: Hyperbolic Attention (hierarchical body structure) === + const hyOut = new Float32Array(channels); + try { + const result = this.rvHyperbolic.compute(mhaOut, tokens, tokens); + for (let c = 0; c < channels; c++) hyOut[c] = result[c]; + } catch (_) { + hyOut.set(mhaOut); + } + + // === Stage 4: Linear Attention (O(n) fast refinement for extremities) === + const linOut = new Float32Array(channels); + try { + const result = this.rvLinear.compute(hyOut, tokens, tokens); + for (let c = 0; c < channels; c++) linOut[c] = result[c]; + } catch (_) { + linOut.set(hyOut); + } + + // === Stage 5: MoE Attention (body-region expert routing) === + const moeOut = new Float32Array(channels); + try { + const result = this.rvMoE.compute(linOut, tokens, tokens); + for (let c = 0; c < channels; c++) moeOut[c] = result[c]; + } catch (_) { + moeOut.set(linOut); + } + + // === Stage 6: Local-Global Attention (detail + context) === + const lgOut = new Float32Array(channels); + try { + const result = this.rvLocalGlobal.compute(moeOut, tokens, tokens); + for (let c = 0; c < channels; c++) lgOut[c] = result[c]; + } catch (_) { + lgOut.set(moeOut); + } + + // === Blend all 6 outputs === + // Use WASM softmax on log-energy scores for dynamic stage weighting + const blended = new Float32Array(channels); + const stages = [flashOut, mhaOut, hyOut, linOut, moeOut, lgOut]; + // Use log-energy to prevent exp() overflow in softmax + const logEnergies = new Float32Array(6); + for (let s = 0; s < 6; s++) { + const e = this._energy(stages[s]); + logEnergies[s] = e > 1e-10 ? Math.log(e) : -20; + } + try { mod.softmax(logEnergies); } catch (_) { + let max = -Infinity; + for (let i = 0; i < 6; i++) max = Math.max(max, logEnergies[i]); + let sum = 0; + for (let i = 0; i < 6; i++) { logEnergies[i] = Math.exp(logEnergies[i] - max); sum += logEnergies[i]; } + for (let i = 0; i < 6; i++) logEnergies[i] /= sum; + } + for (let c = 0; c < channels; c++) { + for (let s = 0; s < 6; s++) { + blended[c] += logEnergies[s] * stages[s][c]; + } + } + + // Batch normalize only when we have enough diversity (skip for single vectors) + // Single-vector batch norm collapses to zeros, killing embedding space + let normed = blended; + + // Project to embeddingDim + const emb = new Float32Array(this.embeddingDim); + for (let o = 0; o < this.embeddingDim; o++) { + let sum = 0; + for (let i = 0; i < channels; i++) { + sum += normed[i] * this.attnProjWeights[i * this.embeddingDim + o]; + } + emb[o] = sum; + } + + // L2 normalize using RuVector WASM + if (this.normalize) { + try { mod.normalize(emb); } catch (_) { + let norm = 0; + for (let i = 0; i < emb.length; i++) norm += emb[i] * emb[i]; + norm = Math.sqrt(norm); + if (norm > 1e-8) for (let i = 0; i < emb.length; i++) emb[i] /= norm; + } + } + + return emb; + } + + /** Compute vector energy (L2 norm squared) for attention weighting */ + _energy(vec) { + let e = 0; + for (let i = 0; i < vec.length; i++) e += vec[i] * vec[i]; + return e; + } + + _conv2d3x3(input, H, W, Cin, Cout) { + const outH = H - 2, outW = W - 2; + const output = new Float32Array(outH * outW * Cout); + for (let y = 0; y < outH; y++) { + for (let x = 0; x < outW; x++) { + for (let co = 0; co < Cout; co++) { + let sum = 0; + for (let ky = 0; ky < 3; ky++) { + for (let kx = 0; kx < 3; kx++) { + for (let ci = 0; ci < Cin; ci++) { + const px = ((y + ky) * W + (x + kx)) * Cin + ci; + const wt = (((ky * 3 + kx) * Cin) + ci) * Cout + co; + sum += input[px] * this.convWeights[wt]; + } + } + } + output[(y * outW + x) * Cout + co] = sum; + } + } + } + return output; + } + + _batchNorm(data, channels) { + const spatial = data.length / channels; + for (let i = 0; i < spatial; i++) { + for (let c = 0; c < channels; c++) { + const idx = i * channels + c; + data[idx] = this.bnGamma[c] * (data[idx] - this.bnMean[c]) / Math.sqrt(this.bnVar[c] + 1e-5) + this.bnBeta[c]; + } + } + } + + _resize(rgbData, srcW, srcH, dstW, dstH) { + const output = new Float32Array(dstW * dstH * 3); + const xRatio = srcW / dstW; + const yRatio = srcH / dstH; + for (let y = 0; y < dstH; y++) { + for (let x = 0; x < dstW; x++) { + const sx = Math.min(Math.floor(x * xRatio), srcW - 1); + const sy = Math.min(Math.floor(y * yRatio), srcH - 1); + const srcIdx = (sy * srcW + sx) * 3; + const dstIdx = (y * dstW + x) * 3; + output[dstIdx] = rgbData[srcIdx] / 255.0; + output[dstIdx + 1] = rgbData[srcIdx + 1] / 255.0; + output[dstIdx + 2] = rgbData[srcIdx + 2] / 255.0; + } + } + return output; + } + + /** Cosine similarity using WASM when available, JS fallback */ + cosineSim(a, b) { + if (this.rvModule) { + try { return this.rvModule.cosine_similarity(a, b); } catch (_) { /* fallback */ } + } + return CnnEmbedder.cosineSimilarity(a, b); + } + + /** L2 norm using WASM when available */ + l2Norm(vec) { + if (this.rvModule) { + try { return this.rvModule.l2_norm(vec); } catch (_) { /* fallback */ } + } + let norm = 0; + for (let i = 0; i < vec.length; i++) norm += vec[i] * vec[i]; + return Math.sqrt(norm); + } + + /** Pairwise distance matrix using WASM (for skeleton validation) */ + pairwiseDistances(vectors) { + if (this.rvModule) { + try { return this.rvModule.pairwise_distances(vectors); } catch (_) { /* fallback */ } + } + return null; + } + + /** Static JS fallback for cosine similarity */ + static cosineSimilarity(a, b) { + let dot = 0, normA = 0, normB = 0; + for (let i = 0; i < a.length; i++) { + dot += a[i] * b[i]; + normA += a[i] * a[i]; + normB += b[i] * b[i]; + } + normA = Math.sqrt(normA); + normB = Math.sqrt(normB); + if (normA < 1e-8 || normB < 1e-8) return 0; + return dot / (normA * normB); + } +} diff --git a/pose-fusion/js/js/csi-simulator.js b/pose-fusion/js/js/csi-simulator.js new file mode 100644 index 00000000..f234d593 --- /dev/null +++ b/pose-fusion/js/js/csi-simulator.js @@ -0,0 +1,363 @@ +/** + * CSI Simulator — Generates realistic WiFi Channel State Information data. + * + * In live mode, connects to the sensing server via WebSocket. + * In demo mode, generates synthetic CSI that correlates with detected motion. + * + * Outputs: 3-channel pseudo-image (amplitude, phase, temporal diff) + * matching the ADR-018 frame format expectations. + */ + +export class CsiSimulator { + static VERSION = 'v4-drift'; // Cache-bust verification + + constructor(opts = {}) { + this.subcarriers = opts.subcarriers || 52; // 802.11n HT20 + this.timeWindow = opts.timeWindow || 56; // frames in sliding window + this.mode = 'demo'; // 'demo' | 'live' + this.ws = null; + + // Circular buffer for CSI frames + this.amplitudeBuffer = []; + this.phaseBuffer = []; + this.frameCount = 0; + + // Noise parameters + this._rng = this._mulberry32(opts.seed || 7); + this._noiseState = new Float32Array(this.subcarriers); + this._baseAmplitude = new Float32Array(this.subcarriers); + this._basePhase = new Float32Array(this.subcarriers); + + // Initialize base CSI profile (empty room) + for (let i = 0; i < this.subcarriers; i++) { + this._baseAmplitude[i] = 0.5 + 0.3 * Math.sin(i * 0.12); + this._basePhase[i] = (i / this.subcarriers) * Math.PI * 2; + } + + // RSSI tracking + this.rssiDbm = -70; // default mid-range + this._rssiTarget = -70; + + // Person influence (updated from video motion) + this.personPresence = 0; + this.personX = 0.5; + this.personY = 0.5; + this.personMotion = 0; + } + + /** + * Connect to live sensing server WebSocket + * @param {string} url - WebSocket URL (e.g. ws://localhost:3030/ws/csi) + */ + async connectLive(url) { + return new Promise((resolve) => { + try { + this.ws = new WebSocket(url); + this.ws.binaryType = 'arraybuffer'; + this.ws.onmessage = (evt) => this._handleLiveFrame(evt.data); + this.ws.onopen = () => { this.mode = 'live'; resolve(true); }; + this.ws.onerror = () => resolve(false); + this.ws.onclose = () => { this.mode = 'demo'; }; + // Timeout after 3s + setTimeout(() => { if (this.mode !== 'live') resolve(false); }, 3000); + } catch { + resolve(false); + } + }); + } + + disconnect() { + if (this.ws) { this.ws.close(); this.ws = null; } + this.mode = 'demo'; + } + + get isLive() { return this.mode === 'live'; } + + /** + * Update person state from video detection (for correlated demo data). + * When person exits frame, CSI maintains presence with slow decay + * (simulating through-wall sensing capability). + */ + updatePersonState(presence, x, y, motion) { + // Don't override real CSI sensing with synthetic video-derived state + if (this.mode === 'live') return; + + if (presence > 0.1) { + // Person detected in video — update CSI state directly + this.personPresence = presence; + this.personX = x; + this.personY = y; + this.personMotion = motion; + this._lastSeenTime = performance.now(); + this._lastSeenX = x; + this._lastSeenY = y; + } else if (this._lastSeenTime) { + // Person NOT in video — CSI "through-wall" persistence + const elapsed = (performance.now() - this._lastSeenTime) / 1000; + // CSI can sense through walls for ~10 seconds with decaying confidence + const decayRate = 0.15; // Lose ~15% per second + this.personPresence = Math.max(0, 1.0 - elapsed * decayRate); + // Position slowly drifts (person walking behind wall) + this.personX = this._lastSeenX; + this.personY = this._lastSeenY; + this.personMotion = Math.max(0, motion * 0.5 + this.personPresence * 0.2); + + if (this.personPresence < 0.05) { + this._lastSeenTime = null; + } + } else { + this.personPresence = 0; + this.personMotion = 0; + } + } + + /** + * Generate next CSI frame (demo mode) or return latest live frame + * @param {number} elapsed - Time in seconds + * @returns {{ amplitude: Float32Array, phase: Float32Array, snr: number }} + */ + nextFrame(elapsed) { + const amp = new Float32Array(this.subcarriers); + const phase = new Float32Array(this.subcarriers); + + if (this.mode === 'live' && this._liveAmplitude) { + amp.set(this._liveAmplitude); + phase.set(this._livePhase); + } else { + this._generateDemoFrame(amp, phase, elapsed); + } + + // Push to circular buffer + this.amplitudeBuffer.push(new Float32Array(amp)); + this.phaseBuffer.push(new Float32Array(phase)); + if (this.amplitudeBuffer.length > this.timeWindow) { + this.amplitudeBuffer.shift(); + this.phaseBuffer.shift(); + } + + // RSSI: smooth toward target (demo mode generates synthetic RSSI) + if (this.mode === 'demo') { + // Simulate RSSI based on person presence and slow drift + this._rssiTarget = -55 - 25 * (1 - this.personPresence) + Math.sin(elapsed * 0.3) * 3; + } + this.rssiDbm += (this._rssiTarget - this.rssiDbm) * 0.1; + + // SNR estimate + let signalPower = 0, noisePower = 0; + for (let i = 0; i < this.subcarriers; i++) { + signalPower += amp[i] * amp[i]; + noisePower += this._noiseState[i] * this._noiseState[i]; + } + const snr = noisePower > 0 ? 10 * Math.log10(signalPower / noisePower) : 30; + + this.frameCount++; + return { amplitude: amp, phase, snr: Math.max(0, Math.min(40, snr)) }; + } + + /** + * Build 3-channel pseudo-image for CNN input + * @param {number} targetSize - Output image dimension (square) + * @returns {Uint8Array} RGB data (targetSize * targetSize * 3) + */ + buildPseudoImage(targetSize = 56) { + const buf = this.amplitudeBuffer; + const pBuf = this.phaseBuffer; + const frames = buf.length; + if (frames < 2) { + return new Uint8Array(targetSize * targetSize * 3); + } + + const rgb = new Uint8Array(targetSize * targetSize * 3); + + for (let y = 0; y < targetSize; y++) { + const fi = Math.min(Math.floor(y / targetSize * frames), frames - 1); + for (let x = 0; x < targetSize; x++) { + const si = Math.min(Math.floor(x / targetSize * this.subcarriers), this.subcarriers - 1); + const idx = (y * targetSize + x) * 3; + + // R: Amplitude (normalized to 0-255) + const ampVal = buf[fi][si]; + rgb[idx] = Math.min(255, Math.max(0, Math.floor(ampVal * 255))); + + // G: Phase (wrapped to 0-255) + const phaseVal = (pBuf[fi][si] % (2 * Math.PI) + 2 * Math.PI) % (2 * Math.PI); + rgb[idx + 1] = Math.floor(phaseVal / (2 * Math.PI) * 255); + + // B: Temporal difference + if (fi > 0) { + const diff = Math.abs(buf[fi][si] - buf[fi - 1][si]); + rgb[idx + 2] = Math.min(255, Math.floor(diff * 500)); + } + } + } + + return rgb; + } + + /** + * Get heatmap data for visualization + * @returns {{ data: Float32Array, width: number, height: number }} + */ + getHeatmapData() { + const frames = this.amplitudeBuffer.length; + const w = this.subcarriers; + const h = Math.min(frames, this.timeWindow); + const data = new Float32Array(w * h); + for (let y = 0; y < h; y++) { + const fi = frames - h + y; + if (fi >= 0 && fi < frames) { + for (let x = 0; x < w; x++) { + data[y * w + x] = this.amplitudeBuffer[fi][x]; + } + } + } + return { data, width: w, height: h }; + } + + // === Private === + + _generateDemoFrame(amp, phase, elapsed) { + const rng = this._rng; + const presence = this.personPresence; + const motion = this.personMotion; + const px = this.personX; + + for (let i = 0; i < this.subcarriers; i++) { + // Base CSI profile (frequency-selective channel) + let a = this._baseAmplitude[i]; + let p = this._basePhase[i] + elapsed * 0.05; + + // Environmental noise (correlated across subcarriers) + this._noiseState[i] = 0.95 * this._noiseState[i] + 0.05 * (rng() * 2 - 1) * 0.03; + a += this._noiseState[i]; + + // Ambient temporal drift (multipath fading even in empty room) + a += 0.06 * Math.sin(elapsed * 0.7 + i * 0.25) + + 0.04 * Math.sin(elapsed * 1.3 - i * 0.18) + + 0.03 * Math.cos(elapsed * 2.1 + i * 0.4); + + // Person-induced CSI perturbation + if (presence > 0.1) { + // Subcarrier-dependent body reflection (Fresnel zone model) + const freqOffset = (i - this.subcarriers * px) / (this.subcarriers * 0.3); + const bodyReflection = presence * 0.25 * Math.exp(-freqOffset * freqOffset); + + // Motion causes amplitude fluctuation + const motionEffect = motion * 0.15 * Math.sin(elapsed * 3.5 + i * 0.3); + + // Breathing modulation (0.2-0.3 Hz) + const breathing = presence * 0.02 * Math.sin(elapsed * 1.5 + i * 0.05); + + a += bodyReflection + motionEffect + breathing; + p += presence * 0.4 * Math.sin(elapsed * 2.1 + i * 0.15); + } + + amp[i] = Math.max(0, Math.min(1, a)); + phase[i] = p; + } + } + + _handleLiveFrame(data) { + // Handle JSON text frames from the sensing server + if (typeof data === 'string') { + try { + const msg = JSON.parse(data); + this._handleJsonFrame(msg); + } catch (_) { /* ignore malformed JSON */ } + return; + } + + // Handle Blob data (convert to ArrayBuffer and re-process) + if (data instanceof Blob) { + data.arrayBuffer().then(ab => this._handleLiveFrame(ab)).catch(() => {}); + return; + } + + // Handle binary ArrayBuffer frames (ADR-018 format) + if (!(data instanceof ArrayBuffer)) return; + const view = new DataView(data); + // Check ADR-018 magic: 0xC5110001 + if (data.byteLength < 20) return; + const magic = view.getUint32(0, true); + if (magic !== 0xC5110001) return; + + const numSub = Math.min(view.getUint16(8, true), this.subcarriers); + this._liveAmplitude = new Float32Array(this.subcarriers); + this._livePhase = new Float32Array(this.subcarriers); + + const headerSize = 20; + for (let i = 0; i < numSub && (headerSize + i * 4 + 3) < data.byteLength; i++) { + const real = view.getInt16(headerSize + i * 4, true); + const imag = view.getInt16(headerSize + i * 4 + 2, true); + this._liveAmplitude[i] = Math.sqrt(real * real + imag * imag) / 2048; + this._livePhase[i] = Math.atan2(imag, real); + } + } + + _handleJsonFrame(msg) { + // Sensing server sends: { type: "sensing_update", nodes: [{ amplitude: [...], subcarrier_count }], classification, features } + this._liveAmplitude = new Float32Array(this.subcarriers); + this._livePhase = new Float32Array(this.subcarriers); + + // Extract amplitude from sensing_update node data + const node = (msg.nodes && msg.nodes[0]) || msg; + const ampArr = node.amplitude || msg.amplitude; + if (ampArr && Array.isArray(ampArr)) { + const n = Math.min(ampArr.length, this.subcarriers); + // Server sends raw amplitude (already magnitude), normalize to 0-1 + let maxAmp = 0; + for (let i = 0; i < n; i++) maxAmp = Math.max(maxAmp, Math.abs(ampArr[i])); + const scale = maxAmp > 0 ? 1.0 / maxAmp : 1.0; + for (let i = 0; i < n; i++) { + this._liveAmplitude[i] = Math.abs(ampArr[i]) * scale; + } + } + + // Phase from node (if available) + const phaseArr = node.phase || msg.phase; + if (phaseArr && Array.isArray(phaseArr)) { + const n = Math.min(phaseArr.length, this.subcarriers); + for (let i = 0; i < n; i++) this._livePhase[i] = phaseArr[i]; + } else if (ampArr) { + // Synthesize phase from amplitude variation (Hilbert-like estimate) + for (let i = 1; i < this.subcarriers; i++) { + this._livePhase[i] = this._livePhase[i - 1] + (this._liveAmplitude[i] - this._liveAmplitude[i - 1]) * Math.PI; + } + } + + // Handle raw I/Q pairs + const iq = node.iq || msg.iq; + if (iq && Array.isArray(iq)) { + const n = Math.min(iq.length / 2, this.subcarriers); + for (let i = 0; i < n; i++) { + const real = iq[i * 2], imag = iq[i * 2 + 1]; + this._liveAmplitude[i] = Math.sqrt(real * real + imag * imag) / 2048; + this._livePhase[i] = Math.atan2(imag, real); + } + } + + // Extract RSSI from node data + if (typeof node.rssi_dbm === 'number') { + this._rssiTarget = node.rssi_dbm; + } else if (msg.features && typeof msg.features.mean_rssi === 'number') { + this._rssiTarget = msg.features.mean_rssi; + } + + // Update presence from server classification + const cls = msg.classification; + if (cls) { + if (typeof cls.confidence === 'number') { + this.personPresence = cls.presence ? cls.confidence : 0; + } + } + } + + _mulberry32(seed) { + return function() { + let t = (seed += 0x6D2B79F5); + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } +} diff --git a/pose-fusion/js/js/fusion-engine.js b/pose-fusion/js/js/fusion-engine.js new file mode 100644 index 00000000..de454182 --- /dev/null +++ b/pose-fusion/js/js/fusion-engine.js @@ -0,0 +1,183 @@ +/** + * FusionEngine — Attention-weighted dual-modal embedding fusion. + * + * Combines visual (camera) and CSI (WiFi) embeddings with dynamic + * confidence gating based on signal quality. + */ + +export class FusionEngine { + /** + * @param {number} embeddingDim + * @param {object} opts + * @param {object} opts.wasmModule - RuVector WASM module for cosine_similarity etc. + */ + constructor(embeddingDim = 128, opts = {}) { + this.embeddingDim = embeddingDim; + this.wasmModule = opts.wasmModule || null; + + // Learnable attention weights (initialized to balanced 0.5) + this.attentionWeights = new Float32Array(embeddingDim).fill(0.5); + + // Dynamic modality confidence [0, 1] + this.videoConfidence = 1.0; + this.csiConfidence = 0.0; + this.fusedConfidence = 0.5; + + // Smoothing for confidence transitions + this._smoothAlpha = 0.85; + + // Embedding history for visualization + this.recentVideoEmbeddings = []; + this.recentCsiEmbeddings = []; + this.recentFusedEmbeddings = []; + this.maxHistory = 50; + } + + /** Set the WASM module reference (called after WASM loads) */ + setWasmModule(mod) { this.wasmModule = mod; } + + /** + * Update quality-based confidence scores + * @param {number} videoBrightness - [0,1] video brightness quality + * @param {number} videoMotion - [0,1] motion detected + * @param {number} csiSnr - CSI signal-to-noise ratio in dB + * @param {boolean} csiActive - Whether CSI source is connected + */ + updateConfidence(videoBrightness, videoMotion, csiSnr, csiActive) { + // Video confidence: drops with low brightness, boosted by motion + let vc = 0; + if (videoBrightness > 0.05) { + vc = Math.min(1, videoBrightness * 1.5) * 0.7 + Math.min(1, videoMotion * 3) * 0.3; + } + + // CSI confidence: based on SNR and connection status + let cc = 0; + if (csiActive) { + cc = Math.min(1, csiSnr / 25); // 25dB = full confidence + } + + // Smooth transitions + this.videoConfidence = this._smoothAlpha * this.videoConfidence + (1 - this._smoothAlpha) * vc; + this.csiConfidence = this._smoothAlpha * this.csiConfidence + (1 - this._smoothAlpha) * cc; + + // Fused confidence is the max of either (fusion can only help) + this.fusedConfidence = Math.min(1, Math.sqrt( + this.videoConfidence * this.videoConfidence + this.csiConfidence * this.csiConfidence + )); + } + + /** + * Fuse video and CSI embeddings + * @param {Float32Array|null} videoEmb - Visual embedding (or null if video-off) + * @param {Float32Array|null} csiEmb - CSI embedding (or null if CSI-off) + * @param {string} mode - 'dual' | 'video' | 'csi' + * @returns {Float32Array} Fused embedding + */ + fuse(videoEmb, csiEmb, mode = 'dual') { + const dim = this.embeddingDim; + const fused = new Float32Array(dim); + + if (mode === 'video' || !csiEmb) { + if (videoEmb) fused.set(videoEmb); + this._recordEmbedding(videoEmb, null, fused); + return fused; + } + + if (mode === 'csi' || !videoEmb) { + if (csiEmb) fused.set(csiEmb); + this._recordEmbedding(null, csiEmb, fused); + return fused; + } + + // Dual mode: attention-weighted fusion with confidence gating + const totalConf = this.videoConfidence + this.csiConfidence; + const videoWeight = totalConf > 0 ? this.videoConfidence / totalConf : 0.5; + + for (let i = 0; i < dim; i++) { + const alpha = this.attentionWeights[i] * videoWeight + + (1 - this.attentionWeights[i]) * (1 - videoWeight); + fused[i] = alpha * videoEmb[i] + (1 - alpha) * csiEmb[i]; + } + + // Re-normalize using WASM when available + if (this.wasmModule) { + try { this.wasmModule.normalize(fused); } catch (_) { this._jsNormalize(fused); } + } else { + this._jsNormalize(fused); + } + + this._recordEmbedding(videoEmb, csiEmb, fused); + return fused; + } + + /** + * Get embedding pairs for 2D visualization (PCA projection) + * @returns {{ video: Array, csi: Array, fused: Array }} + */ + getEmbeddingPoints() { + // Sparse random projection: pick a few dimensions with fixed coefficients + // to get visible 2D spread (avoids cancellation from summing all 128 dims) + const project = (emb) => { + if (!emb || emb.length < 4) return null; + // Use 8 sparse dimensions with predetermined signs (seeded, not random) + const dim = emb.length; + const x = emb[0] * 3.2 - emb[3] * 2.8 + emb[7] * 2.1 - emb[12] * 1.9 + + (dim > 30 ? emb[29] * 1.5 - emb[31] * 1.3 : 0) + + (dim > 60 ? emb[55] * 1.1 - emb[60] * 0.9 : 0); + const y = emb[1] * 3.0 - emb[5] * 2.5 + emb[9] * 2.3 - emb[15] * 1.7 + + (dim > 40 ? emb[37] * 1.4 - emb[42] * 1.2 : 0) + + (dim > 80 ? emb[73] * 1.0 - emb[80] * 0.8 : 0); + return [x, y]; + }; + + return { + video: this.recentVideoEmbeddings.map(project).filter(Boolean), + csi: this.recentCsiEmbeddings.map(project).filter(Boolean), + fused: this.recentFusedEmbeddings.map(project).filter(Boolean) + }; + } + + /** + * Cross-modal similarity score + * @returns {number} Cosine similarity between latest video and CSI embeddings + */ + getCrossModalSimilarity() { + const v = this.recentVideoEmbeddings[this.recentVideoEmbeddings.length - 1]; + const c = this.recentCsiEmbeddings[this.recentCsiEmbeddings.length - 1]; + if (!v || !c) return 0; + + // Use WASM cosine_similarity when available + if (this.wasmModule) { + try { return this.wasmModule.cosine_similarity(v, c); } catch (_) { /* fallback */ } + } + + let dot = 0, na = 0, nb = 0; + for (let i = 0; i < v.length; i++) { + dot += v[i] * c[i]; + na += v[i] * v[i]; + nb += c[i] * c[i]; + } + na = Math.sqrt(na); nb = Math.sqrt(nb); + return (na > 1e-8 && nb > 1e-8) ? dot / (na * nb) : 0; + } + + _jsNormalize(vec) { + let norm = 0; + for (let i = 0; i < vec.length; i++) norm += vec[i] * vec[i]; + norm = Math.sqrt(norm); + if (norm > 1e-8) for (let i = 0; i < vec.length; i++) vec[i] /= norm; + } + + _recordEmbedding(video, csi, fused) { + if (video) { + this.recentVideoEmbeddings.push(new Float32Array(video)); + if (this.recentVideoEmbeddings.length > this.maxHistory) this.recentVideoEmbeddings.shift(); + } + if (csi) { + this.recentCsiEmbeddings.push(new Float32Array(csi)); + if (this.recentCsiEmbeddings.length > this.maxHistory) this.recentCsiEmbeddings.shift(); + } + this.recentFusedEmbeddings.push(new Float32Array(fused)); + if (this.recentFusedEmbeddings.length > this.maxHistory) this.recentFusedEmbeddings.shift(); + } +} diff --git a/pose-fusion/js/js/main.js b/pose-fusion/js/js/main.js new file mode 100644 index 00000000..58348f4b --- /dev/null +++ b/pose-fusion/js/js/main.js @@ -0,0 +1,472 @@ +/** + * RuView — Dual-Modal Pose Estimation Demo + * + * Main orchestration: video capture → CNN embedding → CSI processing → fusion → rendering + */ + +import { VideoCapture } from './video-capture.js?v=13'; +import { CsiSimulator } from './csi-simulator.js?v=13'; +import { CnnEmbedder } from './cnn-embedder.js?v=13'; +import { FusionEngine } from './fusion-engine.js?v=13'; +import { PoseDecoder } from './pose-decoder.js?v=13'; +import { CanvasRenderer } from './canvas-renderer.js?v=13'; + +// === State === +let mode = 'dual'; // 'dual' | 'video' | 'csi' +let isRunning = false; +let isPaused = false; +let startTime = 0; +let frameCount = 0; +let fps = 0; +let lastFpsTime = 0; +let confidenceThreshold = 0.3; + +// Latency tracking +const latency = { video: 0, csi: 0, fusion: 0, total: 0 }; + +// === Components === +const videoCapture = new VideoCapture(document.getElementById('webcam')); +const csiSimulator = new CsiSimulator({ subcarriers: 52, timeWindow: 56 }); +const visualCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 42 }); +const csiCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 137 }); +const fusionEngine = new FusionEngine(128); +const poseDecoder = new PoseDecoder(128); +const renderer = new CanvasRenderer(); + +// === Canvas Elements === +const skeletonCanvas = document.getElementById('skeleton-canvas'); +const skeletonCtx = skeletonCanvas.getContext('2d'); +const csiCanvas = document.getElementById('csi-canvas'); +const csiCtx = csiCanvas.getContext('2d'); +const embeddingCanvas = document.getElementById('embedding-canvas'); +const embeddingCtx = embeddingCanvas.getContext('2d'); + +// === UI Elements === +const modeSelect = document.getElementById('mode-select'); +const statusDot = document.getElementById('status-dot'); +const statusLabel = document.getElementById('status-label'); +const fpsDisplay = document.getElementById('fps-display'); +const cameraPrompt = document.getElementById('camera-prompt'); +const startCameraBtn = document.getElementById('start-camera-btn'); +const pauseBtn = document.getElementById('pause-btn'); +const confSlider = document.getElementById('confidence-slider'); +const confValue = document.getElementById('confidence-value'); +const wsUrlInput = document.getElementById('ws-url'); +const connectWsBtn = document.getElementById('connect-ws-btn'); + +// Fusion bar elements +const videoBar = document.getElementById('video-bar'); +const csiBar = document.getElementById('csi-bar'); +const fusedBar = document.getElementById('fused-bar'); +const videoBarVal = document.getElementById('video-bar-val'); +const csiBarVal = document.getElementById('csi-bar-val'); +const fusedBarVal = document.getElementById('fused-bar-val'); + +// Latency elements +const latVideoEl = document.getElementById('lat-video'); +const latCsiEl = document.getElementById('lat-csi'); +const latFusionEl = document.getElementById('lat-fusion'); +const latTotalEl = document.getElementById('lat-total'); + +// Cross-modal similarity +const crossModalEl = document.getElementById('cross-modal-sim'); + +// RSSI elements +const rssiBarEl = document.getElementById('rssi-bar'); +const rssiValueEl = document.getElementById('rssi-value'); +const rssiQualityEl = document.getElementById('rssi-quality'); +const rssiSparkCanvas = document.getElementById('rssi-sparkline'); +const rssiSparkCtx = rssiSparkCanvas ? rssiSparkCanvas.getContext('2d') : null; +const rssiHistory = []; +const RSSI_HISTORY_MAX = 80; + +// === Initialize === +function init() { + console.log(`[PoseFusion] init() v4 — CsiSimulator=${CsiSimulator.VERSION || 'OLD'}, starting...`); + resizeCanvases(); + console.log(`[PoseFusion] canvases: skeleton=${skeletonCanvas.width}x${skeletonCanvas.height}, csi=${csiCanvas.width}x${csiCanvas.height}, emb=${embeddingCanvas.width}x${embeddingCanvas.height}`); + window.addEventListener('resize', resizeCanvases); + + // Mode change + modeSelect.addEventListener('change', (e) => { + mode = e.target.value; + updateModeUI(); + }); + + // Camera start + startCameraBtn.addEventListener('click', startCamera); + + // Pause + pauseBtn.addEventListener('click', () => { + isPaused = !isPaused; + pauseBtn.textContent = isPaused ? '▶ Resume' : '⏸ Pause'; + pauseBtn.classList.toggle('active', isPaused); + }); + + // Confidence slider + confSlider.addEventListener('input', (e) => { + confidenceThreshold = parseFloat(e.target.value); + confValue.textContent = confidenceThreshold.toFixed(2); + }); + + // WebSocket connect + connectWsBtn.addEventListener('click', async () => { + const url = wsUrlInput.value.trim(); + if (!url) return; + connectWsBtn.textContent = 'Connecting...'; + const ok = await csiSimulator.connectLive(url); + connectWsBtn.textContent = ok ? '✓ Connected' : 'Connect'; + if (ok) { + connectWsBtn.classList.add('active'); + } + }); + + // Try to load RuVector Attention WASM embedders (non-blocking) + const wasmBase = new URL('../pkg/ruvector-attention', import.meta.url).href; + visualCnn.tryLoadWasm(wasmBase).then((ok) => { + // Share the WASM module with FusionEngine for cosine_similarity, normalize, etc. + if (visualCnn.rvModule) fusionEngine.setWasmModule(visualCnn.rvModule); + // Update footer backend label + const backendEl = document.getElementById('cnn-backend'); + if (backendEl) { + backendEl.textContent = ok && visualCnn.useRuVector + ? `RuVector WASM v${visualCnn.rvModule.version()} — 6 attention mechanisms` + : 'ruvector-cnn (JS fallback)'; + } + }); + csiCnn.tryLoadWasm(wasmBase); + + // Auto-connect to local sensing server WebSocket if available + const defaultWsUrl = 'ws://localhost:8765/ws/sensing'; + if (wsUrlInput) wsUrlInput.value = defaultWsUrl; + csiSimulator.connectLive(defaultWsUrl).then(ok => { + if (ok && connectWsBtn) { + connectWsBtn.textContent = '✓ Live ESP32'; + connectWsBtn.classList.add('active'); + statusLabel.textContent = 'LIVE CSI'; + statusDot.classList.remove('offline'); + } + }); + + // Auto-start camera for video/dual modes + updateModeUI(); + startTime = performance.now() / 1000; + isRunning = true; + requestAnimationFrame(mainLoop); +} + +async function startCamera() { + cameraPrompt.style.display = 'none'; + const ok = await videoCapture.start(); + if (ok) { + statusDot.classList.remove('offline'); + statusLabel.textContent = 'LIVE'; + resizeCanvases(); + } else { + cameraPrompt.style.display = 'flex'; + cameraPrompt.querySelector('p').textContent = 'Camera access denied. Try CSI-only mode.'; + } +} + +function updateModeUI() { + const needsVideo = mode !== 'csi'; + + // Show/hide camera prompt + if (needsVideo && !videoCapture.isActive) { + cameraPrompt.style.display = 'flex'; + } else { + cameraPrompt.style.display = 'none'; + } + + // Update mode label in both the overlay and the camera prompt + const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' }; + const modeLabel = document.getElementById('mode-label'); + const promptLabel = document.getElementById('prompt-mode-label'); + if (modeLabel) modeLabel.textContent = labelMap[mode] || mode; + if (promptLabel) promptLabel.textContent = labelMap[mode] || mode; +} + +function resizeCanvases() { + const videoPanel = document.querySelector('.video-panel'); + if (videoPanel) { + const rect = videoPanel.getBoundingClientRect(); + skeletonCanvas.width = rect.width; + skeletonCanvas.height = rect.height; + } + + // CSI canvas (min 200px width) + csiCanvas.width = Math.max(200, csiCanvas.parentElement.clientWidth); + csiCanvas.height = 120; + + // Embedding canvas (min 200px width) + embeddingCanvas.width = Math.max(200, embeddingCanvas.parentElement.clientWidth); + embeddingCanvas.height = 140; +} + +// === Main Loop === +let _loopErrorShown = false; +let _diagDone = false; +function mainLoop(timestamp) { + if (!isRunning) return; + requestAnimationFrame(mainLoop); + + if (isPaused) return; + + try { + const elapsed = performance.now() / 1000 - startTime; + const totalStart = performance.now(); + + // --- Video Pipeline --- + let videoEmb = null; + let motionRegion = null; + if (mode !== 'csi' && videoCapture.isActive) { + const t0 = performance.now(); + const frame = videoCapture.captureFrame(56, 56); + if (frame) { + videoEmb = visualCnn.extract(frame.rgb, frame.width, frame.height); + motionRegion = videoCapture.detectMotionRegion(56, 56); + + // Feed motion to CSI simulator for correlated demo data + // When detected=false, CSI simulator handles through-wall persistence + csiSimulator.updatePersonState( + motionRegion.detected ? 1.0 : 0, + motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5, + motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5, + frame.motion + ); + + fusionEngine.updateConfidence( + frame.brightness, frame.motion, + 0, csiSimulator.isLive || mode === 'dual' + ); + } + latency.video = performance.now() - t0; + } + + // --- CSI Pipeline --- + let csiEmb = null; + if (mode !== 'video') { + const t0 = performance.now(); + const csiFrame = csiSimulator.nextFrame(elapsed); + const pseudoImage = csiSimulator.buildPseudoImage(56); + csiEmb = csiCnn.extract(pseudoImage, 56, 56); + + fusionEngine.updateConfidence( + videoCapture.brightnessScore, + videoCapture.motionScore, + csiFrame.snr, + true + ); + + // Draw CSI heatmap + const heatmap = csiSimulator.getHeatmapData(); + renderer.drawCsiHeatmap(csiCtx, heatmap, csiCanvas.width, csiCanvas.height); + + latency.csi = performance.now() - t0; + } + + // --- Fusion --- + const t0f = performance.now(); + const fusedEmb = fusionEngine.fuse(videoEmb, csiEmb, mode); + latency.fusion = performance.now() - t0f; + + // --- Pose Decode --- + // For CSI-only mode, generate a synthetic motion region from CSI energy + if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) { + const csiPresence = csiSimulator.personPresence; + if (csiPresence > 0.1) { + motionRegion = { + detected: true, + x: 0.25, y: 0.15, w: 0.5, h: 0.7, + coverage: csiPresence, + motionGrid: null, + gridCols: 10, + gridRows: 8 + }; + } + } + + // CSI state for through-wall tracking + const csiState = { + csiPresence: csiSimulator.personPresence, + isLive: csiSimulator.isLive + }; + + const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState); + + // --- Render Skeleton --- + const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' }; + renderer.drawSkeleton(skeletonCtx, keypoints, skeletonCanvas.width, skeletonCanvas.height, { + minConfidence: confidenceThreshold, + color: mode === 'csi' ? 'amber' : 'green', + label: labelMap[mode] + }); + + // --- Render Embedding Space --- + const embPoints = fusionEngine.getEmbeddingPoints(); + renderer.drawEmbeddingSpace(embeddingCtx, embPoints, embeddingCanvas.width, embeddingCanvas.height); + + // --- Update UI --- + latency.total = performance.now() - totalStart; + + // FPS + frameCount++; + if (timestamp - lastFpsTime > 500) { + fps = Math.round(frameCount * 1000 / (timestamp - lastFpsTime)); + lastFpsTime = timestamp; + frameCount = 0; + fpsDisplay.textContent = `${fps} FPS`; + } + + // Fusion bars + const vc = fusionEngine.videoConfidence; + const cc = fusionEngine.csiConfidence; + const fc = fusionEngine.fusedConfidence; + videoBar.style.width = `${vc * 100}%`; + csiBar.style.width = `${cc * 100}%`; + fusedBar.style.width = `${fc * 100}%`; + videoBarVal.textContent = `${Math.round(vc * 100)}%`; + csiBarVal.textContent = `${Math.round(cc * 100)}%`; + fusedBarVal.textContent = `${Math.round(fc * 100)}%`; + + // Latency + latVideoEl.textContent = `${latency.video.toFixed(1)}ms`; + latCsiEl.textContent = `${latency.csi.toFixed(1)}ms`; + latFusionEl.textContent = `${latency.fusion.toFixed(1)}ms`; + latTotalEl.textContent = `${latency.total.toFixed(1)}ms`; + + // Cross-modal similarity + const sim = fusionEngine.getCrossModalSimilarity(); + crossModalEl.textContent = sim.toFixed(3); + + // RuVector attention pipeline stats + const rvStats = poseDecoder.attentionStats; + const rvEnergyEl = document.getElementById('rv-energy'); + const rvRefineEl = document.getElementById('rv-refine'); + const rvImpactEl = document.getElementById('rv-impact'); + if (rvEnergyEl) rvEnergyEl.textContent = (rvStats.energy || 0).toFixed(2); + if (rvRefineEl) rvRefineEl.textContent = ((rvStats.refinementMag || 0) * 1000).toFixed(1) + 'px'; + if (rvImpactEl) { + const impact = Math.min(100, (rvStats.refinementMag || 0) * 5000); + rvImpactEl.textContent = impact.toFixed(0) + '%'; + } + // Pulse the pipeline stages when active + if (visualCnn.useRuVector && rvStats.energy > 0.1) { + document.querySelectorAll('.rv-stage').forEach(el => el.classList.add('active')); + } + + // RSSI update + updateRssi(csiSimulator.rssiDbm); + + // One-time diagnostic + if (!_diagDone) { + _diagDone = true; + console.log(`[PoseFusion] frame 1 OK — mode=${mode}, csi.bufLen=${csiSimulator.amplitudeBuffer.length}, embPts=${embPoints?.fused?.length ?? 0}, rssi=${(csiSimulator.rssiDbm ?? -99).toFixed(1)}`); + } + + } catch (err) { + if (!_loopErrorShown) { + _loopErrorShown = true; + console.error('[MainLoop]', err); + // Show error visually on page + const errDiv = document.createElement('div'); + errDiv.style.cssText = 'position:fixed;bottom:60px;left:24px;right:24px;background:rgba(255,48,64,0.95);color:#fff;padding:12px 16px;border-radius:8px;font:12px/1.4 "JetBrains Mono",monospace;z-index:9999;max-height:120px;overflow:auto'; + errDiv.textContent = `[MainLoop Error] ${err.message}\n${err.stack?.split('\n').slice(0,3).join('\n')}`; + document.body.appendChild(errDiv); + } + } +} + +// === RSSI Visualization === +function updateRssi(dbm) { + if (!rssiBarEl) return; + + // Clamp to typical WiFi range: -100 (worst) to -30 (best) + const clamped = Math.max(-100, Math.min(-30, dbm)); + const pct = ((clamped + 100) / 70) * 100; // 0-100% + + rssiBarEl.style.width = `${pct}%`; + rssiValueEl.textContent = `${Math.round(clamped)} dBm`; + + // Quality label + let quality; + if (clamped > -50) quality = 'Excellent'; + else if (clamped > -60) quality = 'Good'; + else if (clamped > -70) quality = 'Fair'; + else if (clamped > -80) quality = 'Weak'; + else quality = 'Poor'; + rssiQualityEl.textContent = quality; + + // Color the dBm value based on quality + if (clamped > -60) rssiValueEl.style.color = 'var(--green-glow)'; + else if (clamped > -75) rssiValueEl.style.color = 'var(--amber)'; + else rssiValueEl.style.color = 'var(--red-alert)'; + + // Sparkline history + rssiHistory.push(clamped); + if (rssiHistory.length > RSSI_HISTORY_MAX) rssiHistory.shift(); + drawRssiSparkline(); +} + +function drawRssiSparkline() { + if (!rssiSparkCtx || rssiHistory.length < 2) return; + const w = rssiSparkCanvas.width; + const h = rssiSparkCanvas.height; + const ctx = rssiSparkCtx; + + ctx.clearRect(0, 0, w, h); + + // Draw signal strength line + const len = rssiHistory.length; + const step = w / (RSSI_HISTORY_MAX - 1); + + // Gradient fill under line + const grad = ctx.createLinearGradient(0, 0, 0, h); + grad.addColorStop(0, 'rgba(0,210,120,0.3)'); + grad.addColorStop(1, 'rgba(0,210,120,0)'); + + ctx.beginPath(); + for (let i = 0; i < len; i++) { + const x = (RSSI_HISTORY_MAX - len + i) * step; + const y = h - ((rssiHistory[i] + 100) / 70) * h; + if (i === 0) ctx.moveTo(x, y); + else ctx.lineTo(x, y); + } + // Fill area + const lastX = (RSSI_HISTORY_MAX - 1) * step; + const firstX = (RSSI_HISTORY_MAX - len) * step; + ctx.lineTo(lastX, h); + ctx.lineTo(firstX, h); + ctx.closePath(); + ctx.fillStyle = grad; + ctx.fill(); + + // Draw line on top + ctx.beginPath(); + for (let i = 0; i < len; i++) { + const x = (RSSI_HISTORY_MAX - len + i) * step; + const y = h - ((rssiHistory[i] + 100) / 70) * h; + if (i === 0) ctx.moveTo(x, y); + else ctx.lineTo(x, y); + } + ctx.strokeStyle = '#00d878'; + ctx.lineWidth = 1.5; + ctx.stroke(); + + // Pulsing dot at latest value + const latestX = lastX; + const latestY = h - ((rssiHistory[len - 1] + 100) / 70) * h; + const pulse = 0.5 + 0.5 * Math.sin(performance.now() / 300); + ctx.beginPath(); + ctx.arc(latestX, latestY, 2 + pulse, 0, Math.PI * 2); + ctx.fillStyle = '#00d878'; + ctx.fill(); + ctx.beginPath(); + ctx.arc(latestX, latestY, 4 + pulse * 2, 0, Math.PI * 2); + ctx.strokeStyle = `rgba(0,216,120,${0.3 + pulse * 0.3})`; + ctx.lineWidth = 1; + ctx.stroke(); +} + +// Boot +document.addEventListener('DOMContentLoaded', init); diff --git a/pose-fusion/js/js/pose-decoder.js b/pose-fusion/js/js/pose-decoder.js new file mode 100644 index 00000000..338a1ba7 --- /dev/null +++ b/pose-fusion/js/js/pose-decoder.js @@ -0,0 +1,553 @@ +/** + * PoseDecoder — Maps motion detection grid → 17 COCO keypoints. + * + * Uses per-cell motion intensity to track actual body part positions: + * - Head: top-center motion cluster + * - Shoulders/Elbows/Wrists: lateral motion in upper body zone + * - Hips/Knees/Ankles: lower body motion distribution + * + * When person exits frame, CSI data continues tracking (through-wall mode). + */ + +// Extended keypoint definitions: 17 COCO + 9 hand/fingertip approximations = 26 total +export const KEYPOINT_NAMES = [ + 'nose', 'left_eye', 'right_eye', 'left_ear', 'right_ear', + 'left_shoulder', 'right_shoulder', 'left_elbow', 'right_elbow', + 'left_wrist', 'right_wrist', 'left_hip', 'right_hip', + 'left_knee', 'right_knee', 'left_ankle', 'right_ankle', + // Extended: hand keypoints (17-25) + 'left_thumb', 'left_index', 'left_pinky', // 17, 18, 19 + 'right_thumb', 'right_index', 'right_pinky', // 20, 21, 22 + 'left_foot_index', 'right_foot_index', // 23, 24 (toe tips) + 'neck', // 25 (mid-shoulder) +]; + +// Skeleton connections (pairs of keypoint indices) +export const SKELETON_CONNECTIONS = [ + [0, 1], [0, 2], [1, 3], [2, 4], // Head + [0, 25], // Nose → neck + [25, 5], [25, 6], // Neck → shoulders + [5, 7], [7, 9], // Left arm + [6, 8], [8, 10], // Right arm + [5, 11], [6, 12], // Torso + [11, 12], // Hips + [11, 13], [13, 15], // Left leg + [12, 14], [14, 16], // Right leg + // Hand connections + [9, 17], [9, 18], [9, 19], // Left wrist → fingers + [10, 20], [10, 21], [10, 22], // Right wrist → fingers + // Foot connections + [15, 23], [16, 24], // Ankles → toes +]; + +// Standard body proportions (relative to body height) +const PROPORTIONS = { + headToShoulder: 0.15, + shoulderWidth: 0.25, + shoulderToElbow: 0.18, + elbowToWrist: 0.16, + shoulderToHip: 0.30, + hipWidth: 0.18, + hipToKnee: 0.24, + kneeToAnkle: 0.24, + eyeSpacing: 0.04, + earSpacing: 0.07, + // Hand proportions + wristToFinger: 0.09, + fingerSpread: 0.04, + thumbAngle: 0.6, // radians from wrist-elbow axis + // Foot proportions + ankleToToe: 0.06, +}; + +export class PoseDecoder { + constructor(embeddingDim = 128) { + this.embeddingDim = embeddingDim; + this.smoothedKeypoints = null; + this.smoothingFactor = 0.25; // Low = responsive to real movement + this._time = 0; + + // Through-wall tracking state + this._lastBodyState = null; + this._ghostState = null; + this._ghostConfidence = 0; + this._ghostVelocity = { x: 0, y: 0 }; + + // Zone centroid tracking (normalized 0-1 positions) + this._headCx = 0.5; + this._headCy = 0.15; + this._leftArmCx = 0.3; + this._leftArmCy = 0.35; + this._rightArmCx = 0.7; + this._rightArmCy = 0.35; + this._leftLegCx = 0.4; + this._leftLegCy = 0.8; + this._rightLegCx = 0.6; + this._rightLegCy = 0.8; + this._torsoCx = 0.5; + this._torsoCy = 0.45; + + // RuVector embedding → joint mapping + // Each joint gets 2 consecutive embedding dimensions (dx, dy offset) + // and 1 dimension for confidence modulation. 26 joints × 3 = 78 dims used from 128. + // Remaining 50 dims encode global pose features (body scale, rotation, lean). + this._jointEmbMap = this._buildJointEmbeddingMap(embeddingDim); + + // Attention contribution tracking (for UI overlay) + this.attentionStats = { energy: 0, maxDim: 0, refinementMag: 0 }; + } + + /** + * Build the mapping from embedding dimensions to joint refinement signals. + * This maps the RuVector attention output to anatomically meaningful joint offsets. + */ + _buildJointEmbeddingMap(dim) { + const map = []; + // 26 joints × 3 dims each (dx, dy, confidence_mod) = 78 dims + for (let j = 0; j < 26; j++) { + const base = j * 3; + if (base + 2 < dim) { + map.push({ dxDim: base, dyDim: base + 1, confDim: base + 2 }); + } else { + map.push({ dxDim: j % dim, dyDim: (j + 1) % dim, confDim: (j + 2) % dim }); + } + } + // Global pose features from dims 78-127 + return { + joints: map, + scaleDim: Math.min(78, dim - 1), // body scale factor + rotDim: Math.min(79, dim - 1), // body rotation + leanXDim: Math.min(80, dim - 1), // lateral lean + leanYDim: Math.min(81, dim - 1), // forward/back lean + }; + } + + /** + * Decode motion data into 17 keypoints + * @param {Float32Array} embedding - Fused embedding vector + * @param {{ detected, x, y, w, h, motionGrid, gridCols, gridRows, motionCx, motionCy, exitDirection }} motionRegion + * @param {number} elapsed - Time in seconds + * @param {{ csiPresence: number }} csiState - CSI sensing state for through-wall + * @returns {Array<{x: number, y: number, confidence: number, name: string}>} + */ + decode(embedding, motionRegion, elapsed, csiState = {}) { + this._time = elapsed; + + const hasMotion = motionRegion && motionRegion.detected; + const hasCsi = csiState && csiState.csiPresence > 0.1; + + if (hasMotion) { + // Active tracking from video motion grid + this._ghostConfidence = 0; + const rawKeypoints = this._trackFromMotionGrid(motionRegion, embedding, elapsed); + this._lastBodyState = { keypoints: rawKeypoints.map(kp => ({...kp})), time: elapsed }; + + // Track exit velocity + if (motionRegion.exitDirection) { + const speed = 0.008; + this._ghostVelocity = { + x: motionRegion.exitDirection === 'left' ? -speed : motionRegion.exitDirection === 'right' ? speed : 0, + y: motionRegion.exitDirection === 'up' ? -speed : motionRegion.exitDirection === 'down' ? speed : 0 + }; + } + + // Apply temporal smoothing + if (this.smoothedKeypoints && this.smoothedKeypoints.length === rawKeypoints.length) { + const alpha = this.smoothingFactor; + for (let i = 0; i < rawKeypoints.length; i++) { + rawKeypoints[i].x = alpha * this.smoothedKeypoints[i].x + (1 - alpha) * rawKeypoints[i].x; + rawKeypoints[i].y = alpha * this.smoothedKeypoints[i].y + (1 - alpha) * rawKeypoints[i].y; + } + } + + this.smoothedKeypoints = rawKeypoints; + return rawKeypoints; + + } else if (this._lastBodyState && (hasCsi || this._ghostConfidence > 0.05)) { + // Through-wall mode: person left frame but CSI still senses them + return this._trackThroughWall(elapsed, csiState); + + } else if (this.smoothedKeypoints) { + // Fade out + const faded = this.smoothedKeypoints.map(kp => ({ + ...kp, + confidence: kp.confidence * 0.88 + })).filter(kp => kp.confidence > 0.05); + if (faded.length === 0) this.smoothedKeypoints = null; + else this.smoothedKeypoints = faded; + return faded; + } + + return []; + } + + /** + * Track body parts from the motion grid. + * Finds the centroid of motion in each body zone and positions joints there. + */ + _trackFromMotionGrid(region, embedding, elapsed) { + const grid = region.motionGrid; + const cols = region.gridCols || 10; + const rows = region.gridRows || 8; + + // Body bounding box (in normalized 0-1 coords) + const bx = region.x, by = region.y, bw = region.w, bh = region.h; + const cx = bx + bw / 2; + const cy = by + bh / 2; + const bodyH = Math.max(bh, 0.3); + const bodyW = Math.max(bw, 0.15); + + // Find motion centroids per body zone from the grid + if (grid) { + const zones = this._findZoneCentroids(grid, cols, rows, bx, by, bw, bh); + // Smooth with low alpha for responsiveness + const a = 0.3; // 30% old, 70% new → responsive + this._headCx = a * this._headCx + (1 - a) * zones.head.x; + this._headCy = a * this._headCy + (1 - a) * zones.head.y; + this._leftArmCx = a * this._leftArmCx + (1 - a) * zones.leftArm.x; + this._leftArmCy = a * this._leftArmCy + (1 - a) * zones.leftArm.y; + this._rightArmCx= a * this._rightArmCx+ (1 - a) * zones.rightArm.x; + this._rightArmCy= a * this._rightArmCy+ (1 - a) * zones.rightArm.y; + this._leftLegCx = a * this._leftLegCx + (1 - a) * zones.leftLeg.x; + this._leftLegCy = a * this._leftLegCy + (1 - a) * zones.leftLeg.y; + this._rightLegCx= a * this._rightLegCx+ (1 - a) * zones.rightLeg.x; + this._rightLegCy= a * this._rightLegCy+ (1 - a) * zones.rightLeg.y; + this._torsoCx = a * this._torsoCx + (1 - a) * zones.torso.x; + this._torsoCy = a * this._torsoCy + (1 - a) * zones.torso.y; + } + + const P = PROPORTIONS; + + // Breathing (subtle) + const breathe = Math.sin(elapsed * 1.5) * 0.002; + + // === Position joints using tracked centroids === + + // HEAD: tracked centroid (top zone) + const headX = this._headCx; + const headY = this._headCy; + + // TORSO center drives shoulder/hip + const torsoX = this._torsoCx; + const shoulderY = this._torsoCy - bodyH * 0.08 + breathe; + const halfW = P.shoulderWidth * bodyH / 2; + const hipHalfW = P.hipWidth * bodyH / 2; + const hipY = shoulderY + P.shoulderToHip * bodyH; + + // ARMS: elbow + wrist driven toward arm zone centroids + // Left arm: shoulder is fixed, elbow/wrist pulled toward left arm centroid + const lShX = torsoX - halfW; + const lShY = shoulderY; + // Vector from shoulder toward arm centroid + const lArmDx = this._leftArmCx - lShX; + const lArmDy = this._leftArmCy - lShY; + const lArmDist = Math.sqrt(lArmDx * lArmDx + lArmDy * lArmDy) || 0.01; + const lArmNx = lArmDx / lArmDist; + const lArmNy = lArmDy / lArmDist; + // Elbow at shoulderToElbow distance along that direction + const elbowLen = P.shoulderToElbow * bodyH; + const lElbowX = lShX + lArmNx * elbowLen; + const lElbowY = lShY + lArmNy * elbowLen; + // Wrist continues further + const wristLen = P.elbowToWrist * bodyH; + const lWristX = lElbowX + lArmNx * wristLen; + const lWristY = lElbowY + lArmNy * wristLen; + + // Right arm: same approach + const rShX = torsoX + halfW; + const rShY = shoulderY; + const rArmDx = this._rightArmCx - rShX; + const rArmDy = this._rightArmCy - rShY; + const rArmDist = Math.sqrt(rArmDx * rArmDx + rArmDy * rArmDy) || 0.01; + const rArmNx = rArmDx / rArmDist; + const rArmNy = rArmDy / rArmDist; + const rElbowX = rShX + rArmNx * elbowLen; + const rElbowY = rShY + rArmNy * elbowLen; + const rWristX = rElbowX + rArmNx * wristLen; + const rWristY = rElbowY + rArmNy * wristLen; + + // LEGS: knees/ankles pulled toward leg zone centroids + const lHipX = torsoX - hipHalfW; + const rHipX = torsoX + hipHalfW; + const lLegDx = this._leftLegCx - lHipX; + const lLegDy = Math.max(0.05, this._leftLegCy - hipY); // always downward + const lLegDist = Math.sqrt(lLegDx * lLegDx + lLegDy * lLegDy) || 0.01; + const lLegNx = lLegDx / lLegDist; + const lLegNy = lLegDy / lLegDist; + const kneeLen = P.hipToKnee * bodyH; + const ankleLen = P.kneeToAnkle * bodyH; + const lKneeX = lHipX + lLegNx * kneeLen; + const lKneeY = hipY + lLegNy * kneeLen; + const lAnkleX = lKneeX + lLegNx * ankleLen; + const lAnkleY = lKneeY + lLegNy * ankleLen; + + const rLegDx = this._rightLegCx - rHipX; + const rLegDy = Math.max(0.05, this._rightLegCy - hipY); + const rLegDist = Math.sqrt(rLegDx * rLegDx + rLegDy * rLegDy) || 0.01; + const rLegNx = rLegDx / rLegDist; + const rLegNy = rLegDy / rLegDist; + const rKneeX = rHipX + rLegNx * kneeLen; + const rKneeY = hipY + rLegNy * kneeLen; + const rAnkleX = rKneeX + rLegNx * ankleLen; + const rAnkleY = rKneeY + rLegNy * ankleLen; + + // Arm raise amount (for hand openness) + const leftArmRaise = Math.max(0, Math.min(1, (shoulderY - this._leftArmCy) / (bodyH * 0.3))); + const rightArmRaise = Math.max(0, Math.min(1, (shoulderY - this._rightArmCy) / (bodyH * 0.3))); + + // Compute hand finger positions from wrist-elbow axis + const lHandAngle = Math.atan2(lWristY - lElbowY, lWristX - lElbowX); + const rHandAngle = Math.atan2(rWristY - rElbowY, rWristX - rElbowX); + const fingerLen = P.wristToFinger * bodyH; + const fingerSpr = P.fingerSpread * bodyH; + + // Hand openness driven by arm raise + arm lateral spread + const lArmSpread = Math.abs(this._leftArmCx - (bx + bw * 0.3)) / (bw * 0.3); + const rArmSpread = Math.abs(this._rightArmCx - (bx + bw * 0.7)) / (bw * 0.3); + const lHandOpen = Math.min(1, leftArmRaise * 0.5 + lArmSpread * 0.5); + const rHandOpen = Math.min(1, rightArmRaise * 0.5 + rArmSpread * 0.5); + + const keypoints = [ + // 0: nose + { x: headX, y: headY + 0.01, confidence: 0.92 }, + // 1: left_eye + { x: headX - P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 }, + // 2: right_eye + { x: headX + P.eyeSpacing * bodyH, y: headY - 0.005, confidence: 0.88 }, + // 3: left_ear + { x: headX - P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, + // 4: right_ear + { x: headX + P.earSpacing * bodyH, y: headY + 0.005, confidence: 0.72 }, + // 5: left_shoulder + { x: lShX, y: lShY, confidence: 0.94 }, + // 6: right_shoulder + { x: rShX, y: rShY, confidence: 0.94 }, + // 7: left_elbow + { x: lElbowX, y: lElbowY, confidence: 0.87 }, + // 8: right_elbow + { x: rElbowX, y: rElbowY, confidence: 0.87 }, + // 9: left_wrist + { x: lWristX, y: lWristY, confidence: 0.82 }, + // 10: right_wrist + { x: rWristX, y: rWristY, confidence: 0.82 }, + // 11: left_hip + { x: lHipX, y: hipY, confidence: 0.91 }, + // 12: right_hip + { x: rHipX, y: hipY, confidence: 0.91 }, + // 13: left_knee + { x: lKneeX, y: lKneeY, confidence: 0.88 }, + // 14: right_knee + { x: rKneeX, y: rKneeY, confidence: 0.88 }, + // 15: left_ankle + { x: lAnkleX, y: lAnkleY, confidence: 0.83 }, + // 16: right_ankle + { x: rAnkleX, y: rAnkleY, confidence: 0.83 }, + + // === Extended keypoints (17-25) === + + // 17: left_thumb — offset at thumb angle from wrist-elbow axis + { x: lWristX + fingerLen * Math.cos(lHandAngle + P.thumbAngle) * (0.6 + lHandOpen * 0.4), + y: lWristY + fingerLen * Math.sin(lHandAngle + P.thumbAngle) * (0.6 + lHandOpen * 0.4), + confidence: 0.68 * (0.5 + lHandOpen * 0.5) }, + // 18: left_index — extends along wrist-elbow axis + { x: lWristX + fingerLen * Math.cos(lHandAngle) + fingerSpr * lHandOpen * Math.cos(lHandAngle + 0.3), + y: lWristY + fingerLen * Math.sin(lHandAngle) + fingerSpr * lHandOpen * Math.sin(lHandAngle + 0.3), + confidence: 0.72 * (0.5 + lHandOpen * 0.5) }, + // 19: left_pinky — offset opposite thumb + { x: lWristX + fingerLen * 0.85 * Math.cos(lHandAngle - P.thumbAngle * 0.7), + y: lWristY + fingerLen * 0.85 * Math.sin(lHandAngle - P.thumbAngle * 0.7), + confidence: 0.60 * (0.5 + lHandOpen * 0.5) }, + + // 20: right_thumb + { x: rWristX + fingerLen * Math.cos(rHandAngle - P.thumbAngle) * (0.6 + rHandOpen * 0.4), + y: rWristY + fingerLen * Math.sin(rHandAngle - P.thumbAngle) * (0.6 + rHandOpen * 0.4), + confidence: 0.68 * (0.5 + rHandOpen * 0.5) }, + // 21: right_index + { x: rWristX + fingerLen * Math.cos(rHandAngle) + fingerSpr * rHandOpen * Math.cos(rHandAngle - 0.3), + y: rWristY + fingerLen * Math.sin(rHandAngle) + fingerSpr * rHandOpen * Math.sin(rHandAngle - 0.3), + confidence: 0.72 * (0.5 + rHandOpen * 0.5) }, + // 22: right_pinky + { x: rWristX + fingerLen * 0.85 * Math.cos(rHandAngle + P.thumbAngle * 0.7), + y: rWristY + fingerLen * 0.85 * Math.sin(rHandAngle + P.thumbAngle * 0.7), + confidence: 0.60 * (0.5 + rHandOpen * 0.5) }, + + // 23: left_foot_index (toe tip) — extends forward from ankle + { x: lAnkleX + P.ankleToToe * bodyH * 0.5, + y: lAnkleY + P.ankleToToe * bodyH * 0.3, + confidence: 0.65 }, + // 24: right_foot_index + { x: rAnkleX + P.ankleToToe * bodyH * 0.5, + y: rAnkleY + P.ankleToToe * bodyH * 0.3, + confidence: 0.65 }, + + // 25: neck (midpoint between shoulders, slightly above) + { x: (lShX + rShX) / 2, y: shoulderY - P.headToShoulder * bodyH * 0.35, confidence: 0.93 }, + ]; + + for (let i = 0; i < keypoints.length; i++) { + keypoints[i].name = KEYPOINT_NAMES[i]; + } + + // === RuVector Attention Embedding Refinement === + // Compute attention stats for the UI pipeline display, but only apply + // positional refinement when a trained model is loaded (random-weight + // embeddings carry no meaningful spatial signal and distort the skeleton). + if (embedding && embedding.length >= 26 * 3) { + this._computeEmbeddingStats(keypoints, embedding, bodyH); + } + + return keypoints; + } + + /** + * Apply RuVector attention embedding to refine joint positions and confidence. + * + * The 128-dim fused embedding is decoded as: + * - Dims 0-77: Per-joint (dx, dy, confidence_mod) × 26 joints + * - Dims 78-81: Global pose parameters (scale, rotation, lean) + * - Dims 82-127: Reserved for cross-modal fusion features + * + * The attention mechanism determines HOW MUCH each spatial region contributes + * to each joint's refinement. Multi-Head captures global relationships, + * Hyperbolic captures hierarchical (torso→limb→hand) dependencies, + * MoE routes different body regions to specialized experts, + * Linear provides fast extremity refinement, Local-Global balances detail/context. + */ + /** + * Compute embedding statistics for UI display without modifying joint positions. + * The 6-stage attention pipeline stats are shown in the RuVector panel. + * Position refinement is disabled until a trained model replaces random weights. + */ + _computeEmbeddingStats(keypoints, emb, bodyH) { + const map = this._jointEmbMap; + const tc = (v) => Math.tanh(Number(v) || 0); + + // Embedding energy (L2 norm of the used dims) + let energy = 0; + for (let i = 0; i < Math.min(emb.length, 82); i++) { + energy += emb[i] * emb[i]; + } + energy = Math.sqrt(energy); + + // Simulated per-joint refinement magnitude (what WOULD be applied) + const scale = bodyH * 0.015; + let totalRefinement = 0; + let maxDimVal = 0; + + for (let j = 0; j < Math.min(keypoints.length, 26); j++) { + const jmap = map.joints[j]; + if (!jmap) continue; + const dx = tc(emb[jmap.dxDim]) * scale; + const dy = tc(emb[jmap.dyDim]) * scale; + totalRefinement += Math.sqrt(dx * dx + dy * dy); + maxDimVal = Math.max(maxDimVal, Math.abs(tc(emb[jmap.dxDim])), Math.abs(tc(emb[jmap.dyDim]))); + } + + this.attentionStats.energy = energy; + this.attentionStats.maxDim = maxDimVal; + this.attentionStats.refinementMag = totalRefinement / 26; + } + + /** + * Find weighted motion centroids for each body zone. + * Divides the bounding box into 6 zones: head, left arm, right arm, torso, left leg, right leg. + * Returns the (x,y) centroid of motion intensity for each zone. + */ + _findZoneCentroids(grid, cols, rows, bx, by, bw, bh) { + // Zone definitions (in grid-relative fractions) + const zones = { + head: { rMin: 0, rMax: 0.2, cMin: 0.25, cMax: 0.75, wx: 0, wy: 0, wt: 0 }, + leftArm: { rMin: 0.1, rMax: 0.6, cMin: 0, cMax: 0.35, wx: 0, wy: 0, wt: 0 }, + rightArm: { rMin: 0.1, rMax: 0.6, cMin: 0.65, cMax: 1.0, wx: 0, wy: 0, wt: 0 }, + torso: { rMin: 0.15, rMax: 0.55, cMin: 0.3, cMax: 0.7, wx: 0, wy: 0, wt: 0 }, + leftLeg: { rMin: 0.5, rMax: 1.0, cMin: 0.1, cMax: 0.5, wx: 0, wy: 0, wt: 0 }, + rightLeg: { rMin: 0.5, rMax: 1.0, cMin: 0.5, cMax: 0.9, wx: 0, wy: 0, wt: 0 }, + }; + + // Accumulate weighted centroids per zone + for (let r = 0; r < rows; r++) { + const ry = r / rows; // 0-1 within grid + for (let c = 0; c < cols; c++) { + const cx_g = c / cols; // 0-1 within grid + const val = grid[r][c]; + if (val < 0.005) continue; // skip near-zero motion + + // Map grid position to body-space coordinates (0-1) + const worldX = bx + cx_g * bw; + const worldY = by + ry * bh; + + // Assign to matching zones (a cell can contribute to multiple overlapping zones) + for (const z of Object.values(zones)) { + if (ry >= z.rMin && ry < z.rMax && cx_g >= z.cMin && cx_g < z.cMax) { + z.wx += worldX * val; + z.wy += worldY * val; + z.wt += val; + } + } + } + } + + // Compute centroids with fallback defaults + const centroid = (z, defX, defY) => ({ + x: z.wt > 0.01 ? z.wx / z.wt : defX, + y: z.wt > 0.01 ? z.wy / z.wt : defY, + weight: z.wt + }); + + const midX = bx + bw / 2; + const midY = by + bh / 2; + + return { + head: centroid(zones.head, midX, by + bh * 0.1), + leftArm: centroid(zones.leftArm, bx + bw * 0.2, midY - bh * 0.05), + rightArm: centroid(zones.rightArm, bx + bw * 0.8, midY - bh * 0.05), + torso: centroid(zones.torso, midX, midY), + leftLeg: centroid(zones.leftLeg, bx + bw * 0.35,by + bh * 0.75), + rightLeg: centroid(zones.rightLeg, bx + bw * 0.65,by + bh * 0.75), + }; + } + + /** + * Through-wall tracking: continue showing pose via CSI when person left video frame. + * The skeleton drifts in the exit direction with decreasing confidence. + */ + _trackThroughWall(elapsed, csiState) { + if (!this._lastBodyState) return []; + + const dt = elapsed - this._lastBodyState.time; + const csiPresence = csiState.csiPresence || 0; + + // Initialize ghost on first call + if (this._ghostConfidence <= 0.05) { + this._ghostConfidence = 0.8; + this._ghostState = this._lastBodyState.keypoints.map(kp => ({...kp})); + } + + // Ghost confidence decays, but CSI presence sustains it + const csiBoost = Math.min(0.7, csiPresence * 0.8); + this._ghostConfidence = Math.max(0.05, this._ghostConfidence * 0.995 - 0.001 + csiBoost * 0.002); + + // Drift the ghost in exit direction + const vx = this._ghostVelocity.x; + const vy = this._ghostVelocity.y; + + // Breathing continues via CSI + const breathe = Math.sin(elapsed * 1.5) * 0.003 * csiPresence; + + const keypoints = this._ghostState.map((kp, i) => { + return { + x: kp.x + vx * dt * 0.3, + y: kp.y + vy * dt * 0.3 + (i >= 5 && i <= 6 ? breathe : 0), + confidence: kp.confidence * this._ghostConfidence * (0.5 + csiPresence * 0.5), + name: kp.name + }; + }); + + // Slow down drift over time + this._ghostVelocity.x *= 0.998; + this._ghostVelocity.y *= 0.998; + + this.smoothedKeypoints = keypoints; + return keypoints; + } +} diff --git a/pose-fusion/js/js/video-capture.js b/pose-fusion/js/js/video-capture.js new file mode 100644 index 00000000..fe3ed333 --- /dev/null +++ b/pose-fusion/js/js/video-capture.js @@ -0,0 +1,235 @@ +/** + * VideoCapture — getUserMedia webcam capture with frame extraction. + * Provides quality metrics (brightness, motion) for fusion confidence gating. + */ + +export class VideoCapture { + constructor(videoElement) { + this.video = videoElement; + this.stream = null; + this.offscreen = document.createElement('canvas'); + this.offCtx = this.offscreen.getContext('2d', { willReadFrequently: true }); + this.prevFrame = null; + this.motionScore = 0; + this.brightnessScore = 0; + } + + async start(constraints = {}) { + const defaultConstraints = { + video: { + width: { ideal: 640 }, + height: { ideal: 480 }, + facingMode: 'user', + frameRate: { ideal: 30 } + }, + audio: false + }; + + try { + this.stream = await navigator.mediaDevices.getUserMedia( + Object.keys(constraints).length ? constraints : defaultConstraints + ); + this.video.srcObject = this.stream; + await this.video.play(); + + this.offscreen.width = this.video.videoWidth; + this.offscreen.height = this.video.videoHeight; + + return true; + } catch (err) { + console.error('[Video] Camera access failed:', err.message); + return false; + } + } + + stop() { + if (this.stream) { + this.stream.getTracks().forEach(t => t.stop()); + this.stream = null; + } + this.video.srcObject = null; + } + + get isActive() { + return this.stream !== null && this.video.readyState >= 2; + } + + get width() { return this.video.videoWidth || 640; } + get height() { return this.video.videoHeight || 480; } + + /** + * Capture current frame as RGB Uint8Array + compute quality metrics. + * @param {number} targetW - Target width for CNN input + * @param {number} targetH - Target height for CNN input + * @returns {{ rgb: Uint8Array, width: number, height: number, motion: number, brightness: number }} + */ + captureFrame(targetW = 56, targetH = 56) { + if (!this.isActive) return null; + + // Draw to offscreen at target resolution + this.offscreen.width = targetW; + this.offscreen.height = targetH; + this.offCtx.drawImage(this.video, 0, 0, targetW, targetH); + const imageData = this.offCtx.getImageData(0, 0, targetW, targetH); + const rgba = imageData.data; + + // Convert RGBA → RGB + const pixels = targetW * targetH; + const rgb = new Uint8Array(pixels * 3); + let brightnessSum = 0; + let motionSum = 0; + + for (let i = 0; i < pixels; i++) { + const r = rgba[i * 4]; + const g = rgba[i * 4 + 1]; + const b = rgba[i * 4 + 2]; + rgb[i * 3] = r; + rgb[i * 3 + 1] = g; + rgb[i * 3 + 2] = b; + + // Luminance for brightness + const lum = 0.299 * r + 0.587 * g + 0.114 * b; + brightnessSum += lum; + + // Motion: diff from previous frame + if (this.prevFrame) { + const pr = this.prevFrame[i * 3]; + const pg = this.prevFrame[i * 3 + 1]; + const pb = this.prevFrame[i * 3 + 2]; + motionSum += Math.abs(r - pr) + Math.abs(g - pg) + Math.abs(b - pb); + } + } + + this.brightnessScore = brightnessSum / (pixels * 255); + this.motionScore = this.prevFrame ? Math.min(1, motionSum / (pixels * 100)) : 0; + this.prevFrame = new Uint8Array(rgb); + + return { + rgb, + width: targetW, + height: targetH, + motion: this.motionScore, + brightness: this.brightnessScore + }; + } + + /** + * Capture full-resolution RGBA for overlay rendering + * @returns {ImageData|null} + */ + captureFullFrame() { + if (!this.isActive) return null; + this.offscreen.width = this.width; + this.offscreen.height = this.height; + this.offCtx.drawImage(this.video, 0, 0); + return this.offCtx.getImageData(0, 0, this.width, this.height); + } + + /** + * Detect motion region + detailed motion grid for body-part tracking. + * Returns bounding box + a grid showing WHERE motion is concentrated. + * @returns {{ x, y, w, h, detected: boolean, motionGrid: number[][], gridCols: number, gridRows: number, exitDirection: string|null }} + */ + detectMotionRegion(targetW = 56, targetH = 56) { + if (!this.isActive || !this.prevFrame) return { detected: false, motionGrid: null }; + + this.offscreen.width = targetW; + this.offscreen.height = targetH; + this.offCtx.drawImage(this.video, 0, 0, targetW, targetH); + const rgba = this.offCtx.getImageData(0, 0, targetW, targetH).data; + + let minX = targetW, minY = targetH, maxX = 0, maxY = 0; + let motionPixels = 0; + const threshold = 25; + + // Motion grid: divide frame into cells and track motion intensity per cell + const gridCols = 10; + const gridRows = 8; + const cellW = targetW / gridCols; + const cellH = targetH / gridRows; + const motionGrid = Array.from({ length: gridRows }, () => new Float32Array(gridCols)); + const cellPixels = cellW * cellH; + + // Also track motion centroid weighted by intensity + let motionCxSum = 0, motionCySum = 0, motionWeightSum = 0; + + for (let y = 0; y < targetH; y++) { + for (let x = 0; x < targetW; x++) { + const i = y * targetW + x; + const r = rgba[i * 4], g = rgba[i * 4 + 1], b = rgba[i * 4 + 2]; + const pr = this.prevFrame[i * 3], pg = this.prevFrame[i * 3 + 1], pb = this.prevFrame[i * 3 + 2]; + const diff = Math.abs(r - pr) + Math.abs(g - pg) + Math.abs(b - pb); + + if (diff > threshold * 3) { + motionPixels++; + if (x < minX) minX = x; + if (y < minY) minY = y; + if (x > maxX) maxX = x; + if (y > maxY) maxY = y; + } + + // Accumulate per-cell motion intensity + const gc = Math.min(Math.floor(x / cellW), gridCols - 1); + const gr = Math.min(Math.floor(y / cellH), gridRows - 1); + const intensity = diff / (3 * 255); // Normalize 0-1 + motionGrid[gr][gc] += intensity / cellPixels; + + // Weighted centroid + if (diff > threshold) { + motionCxSum += x * diff; + motionCySum += y * diff; + motionWeightSum += diff; + } + } + } + + const detected = motionPixels > (targetW * targetH * 0.02); + + // Motion centroid (normalized 0-1) + const motionCx = motionWeightSum > 0 ? motionCxSum / (motionWeightSum * targetW) : 0.5; + const motionCy = motionWeightSum > 0 ? motionCySum / (motionWeightSum * targetH) : 0.5; + + // Detect exit direction: if centroid is near edges + let exitDirection = null; + if (detected && motionCx < 0.1) exitDirection = 'left'; + else if (detected && motionCx > 0.9) exitDirection = 'right'; + else if (detected && motionCy < 0.1) exitDirection = 'up'; + else if (detected && motionCy > 0.9) exitDirection = 'down'; + + // Track last known position for through-wall persistence + if (detected) { + this._lastDetected = { + x: minX / targetW, + y: minY / targetH, + w: (maxX - minX) / targetW, + h: (maxY - minY) / targetH, + cx: motionCx, + cy: motionCy, + exitDirection, + time: performance.now() + }; + } + + return { + detected, + x: minX / targetW, + y: minY / targetH, + w: (maxX - minX) / targetW, + h: (maxY - minY) / targetH, + coverage: motionPixels / (targetW * targetH), + motionGrid, + gridCols, + gridRows, + motionCx, + motionCy, + exitDirection + }; + } + + /** + * Get the last known detection info (for through-wall persistence) + */ + get lastDetection() { + return this._lastDetected || null; + } +} diff --git a/ui/pose-fusion.html b/ui/pose-fusion.html index 08ff952a..39cacc23 100644 --- a/ui/pose-fusion.html +++ b/ui/pose-fusion.html @@ -3,7 +3,7 @@ - WiFi-DensePose — Dual-Modal Pose Estimation + RuView — Dual-Modal Pose Estimation @@ -11,7 +11,7 @@
- +
Dual-Modal Pose Estimation — Live Video + WiFi CSI Fusion
@@ -184,11 +184,11 @@
- WiFi-DensePose · Dual-Modal Pose Estimation · + RuView · Dual-Modal Pose Estimation · Architecture: Conv2D → RuVector 6-Stage Attention (Flash+MHA+Hyperbolic+Linear+MoE+L/G) → Fusion → 26-Keypoint Pose
- GitHub · + GitHub · CNN: ruvector-cnn (loading…) · Observatory
diff --git a/ui/pose-fusion/css/style.css b/ui/pose-fusion/css/style.css index c2a32341..bdf71704 100644 --- a/ui/pose-fusion/css/style.css +++ b/ui/pose-fusion/css/style.css @@ -1,4 +1,4 @@ -/* WiFi-DensePose — Dual-Modal Pose Fusion Demo +/* RuView — Dual-Modal Pose Fusion Demo Dark theme matching Observatory */ @import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;600;700&family=JetBrains+Mono:wght@400;600&display=swap'); diff --git a/ui/pose-fusion/js/main.js b/ui/pose-fusion/js/main.js index 32374ab6..58348f4b 100644 --- a/ui/pose-fusion/js/main.js +++ b/ui/pose-fusion/js/main.js @@ -1,5 +1,5 @@ /** - * WiFi-DensePose — Dual-Modal Pose Estimation Demo + * RuView — Dual-Modal Pose Estimation Demo * * Main orchestration: video capture → CNN embedding → CSI processing → fusion → rendering */