316 lines
10 KiB
JavaScript
316 lines
10 KiB
JavaScript
/**
|
|
* WiFi-DensePose — Dual-Modal Pose Estimation Demo
|
|
*
|
|
* Main orchestration: video capture → CNN embedding → CSI processing → fusion → rendering
|
|
*/
|
|
|
|
import { VideoCapture } from './video-capture.js';
|
|
import { CsiSimulator } from './csi-simulator.js';
|
|
import { CnnEmbedder } from './cnn-embedder.js';
|
|
import { FusionEngine } from './fusion-engine.js';
|
|
import { PoseDecoder } from './pose-decoder.js';
|
|
import { CanvasRenderer } from './canvas-renderer.js';
|
|
|
|
// === State ===
|
|
let mode = 'dual'; // 'dual' | 'video' | 'csi'
|
|
let isRunning = false;
|
|
let isPaused = false;
|
|
let startTime = 0;
|
|
let frameCount = 0;
|
|
let fps = 0;
|
|
let lastFpsTime = 0;
|
|
let confidenceThreshold = 0.3;
|
|
|
|
// Latency tracking
|
|
const latency = { video: 0, csi: 0, fusion: 0, total: 0 };
|
|
|
|
// === Components ===
|
|
const videoCapture = new VideoCapture(document.getElementById('webcam'));
|
|
const csiSimulator = new CsiSimulator({ subcarriers: 52, timeWindow: 56 });
|
|
const visualCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 42 });
|
|
const csiCnn = new CnnEmbedder({ inputSize: 56, embeddingDim: 128, seed: 137 });
|
|
const fusionEngine = new FusionEngine(128);
|
|
const poseDecoder = new PoseDecoder(128);
|
|
const renderer = new CanvasRenderer();
|
|
|
|
// === Canvas Elements ===
|
|
const skeletonCanvas = document.getElementById('skeleton-canvas');
|
|
const skeletonCtx = skeletonCanvas.getContext('2d');
|
|
const csiCanvas = document.getElementById('csi-canvas');
|
|
const csiCtx = csiCanvas.getContext('2d');
|
|
const embeddingCanvas = document.getElementById('embedding-canvas');
|
|
const embeddingCtx = embeddingCanvas.getContext('2d');
|
|
|
|
// === UI Elements ===
|
|
const modeSelect = document.getElementById('mode-select');
|
|
const statusDot = document.getElementById('status-dot');
|
|
const statusLabel = document.getElementById('status-label');
|
|
const fpsDisplay = document.getElementById('fps-display');
|
|
const cameraPrompt = document.getElementById('camera-prompt');
|
|
const startCameraBtn = document.getElementById('start-camera-btn');
|
|
const pauseBtn = document.getElementById('pause-btn');
|
|
const confSlider = document.getElementById('confidence-slider');
|
|
const confValue = document.getElementById('confidence-value');
|
|
const wsUrlInput = document.getElementById('ws-url');
|
|
const connectWsBtn = document.getElementById('connect-ws-btn');
|
|
|
|
// Fusion bar elements
|
|
const videoBar = document.getElementById('video-bar');
|
|
const csiBar = document.getElementById('csi-bar');
|
|
const fusedBar = document.getElementById('fused-bar');
|
|
const videoBarVal = document.getElementById('video-bar-val');
|
|
const csiBarVal = document.getElementById('csi-bar-val');
|
|
const fusedBarVal = document.getElementById('fused-bar-val');
|
|
|
|
// Latency elements
|
|
const latVideoEl = document.getElementById('lat-video');
|
|
const latCsiEl = document.getElementById('lat-csi');
|
|
const latFusionEl = document.getElementById('lat-fusion');
|
|
const latTotalEl = document.getElementById('lat-total');
|
|
|
|
// Cross-modal similarity
|
|
const crossModalEl = document.getElementById('cross-modal-sim');
|
|
|
|
// === Initialize ===
|
|
function init() {
|
|
resizeCanvases();
|
|
window.addEventListener('resize', resizeCanvases);
|
|
|
|
// Mode change
|
|
modeSelect.addEventListener('change', (e) => {
|
|
mode = e.target.value;
|
|
updateModeUI();
|
|
});
|
|
|
|
// Camera start
|
|
startCameraBtn.addEventListener('click', startCamera);
|
|
|
|
// Pause
|
|
pauseBtn.addEventListener('click', () => {
|
|
isPaused = !isPaused;
|
|
pauseBtn.textContent = isPaused ? '▶ Resume' : '⏸ Pause';
|
|
pauseBtn.classList.toggle('active', isPaused);
|
|
});
|
|
|
|
// Confidence slider
|
|
confSlider.addEventListener('input', (e) => {
|
|
confidenceThreshold = parseFloat(e.target.value);
|
|
confValue.textContent = confidenceThreshold.toFixed(2);
|
|
});
|
|
|
|
// WebSocket connect
|
|
connectWsBtn.addEventListener('click', async () => {
|
|
const url = wsUrlInput.value.trim();
|
|
if (!url) return;
|
|
connectWsBtn.textContent = 'Connecting...';
|
|
const ok = await csiSimulator.connectLive(url);
|
|
connectWsBtn.textContent = ok ? '✓ Connected' : 'Connect';
|
|
if (ok) {
|
|
connectWsBtn.classList.add('active');
|
|
}
|
|
});
|
|
|
|
// Try to load WASM embedders (non-blocking)
|
|
// Resolve relative to this JS module file (in pose-fusion/js/) → ../pkg/
|
|
const wasmBase = new URL('../pkg/ruvector_cnn_wasm', import.meta.url).href;
|
|
visualCnn.tryLoadWasm(wasmBase);
|
|
csiCnn.tryLoadWasm(wasmBase);
|
|
|
|
// Auto-connect to local sensing server WebSocket if available
|
|
const defaultWsUrl = 'ws://localhost:8765/ws/sensing';
|
|
if (wsUrlInput) wsUrlInput.value = defaultWsUrl;
|
|
csiSimulator.connectLive(defaultWsUrl).then(ok => {
|
|
if (ok && connectWsBtn) {
|
|
connectWsBtn.textContent = '✓ Live ESP32';
|
|
connectWsBtn.classList.add('active');
|
|
statusLabel.textContent = 'LIVE CSI';
|
|
statusDot.classList.remove('offline');
|
|
}
|
|
});
|
|
|
|
// Auto-start camera for video/dual modes
|
|
updateModeUI();
|
|
startTime = performance.now() / 1000;
|
|
isRunning = true;
|
|
requestAnimationFrame(mainLoop);
|
|
}
|
|
|
|
async function startCamera() {
|
|
cameraPrompt.style.display = 'none';
|
|
const ok = await videoCapture.start();
|
|
if (ok) {
|
|
statusDot.classList.remove('offline');
|
|
statusLabel.textContent = 'LIVE';
|
|
resizeCanvases();
|
|
} else {
|
|
cameraPrompt.style.display = 'flex';
|
|
cameraPrompt.querySelector('p').textContent = 'Camera access denied. Try CSI-only mode.';
|
|
}
|
|
}
|
|
|
|
function updateModeUI() {
|
|
const needsVideo = mode !== 'csi';
|
|
const needsCsi = mode !== 'video';
|
|
|
|
// Show/hide camera prompt
|
|
if (needsVideo && !videoCapture.isActive) {
|
|
cameraPrompt.style.display = 'flex';
|
|
} else {
|
|
cameraPrompt.style.display = 'none';
|
|
}
|
|
}
|
|
|
|
function resizeCanvases() {
|
|
const videoPanel = document.querySelector('.video-panel');
|
|
if (videoPanel) {
|
|
const rect = videoPanel.getBoundingClientRect();
|
|
skeletonCanvas.width = rect.width;
|
|
skeletonCanvas.height = rect.height;
|
|
}
|
|
|
|
// CSI canvas
|
|
csiCanvas.width = csiCanvas.parentElement.clientWidth;
|
|
csiCanvas.height = 120;
|
|
|
|
// Embedding canvas
|
|
embeddingCanvas.width = embeddingCanvas.parentElement.clientWidth;
|
|
embeddingCanvas.height = 140;
|
|
}
|
|
|
|
// === Main Loop ===
|
|
function mainLoop(timestamp) {
|
|
if (!isRunning) return;
|
|
requestAnimationFrame(mainLoop);
|
|
|
|
if (isPaused) return;
|
|
|
|
const elapsed = performance.now() / 1000 - startTime;
|
|
const totalStart = performance.now();
|
|
|
|
// --- Video Pipeline ---
|
|
let videoEmb = null;
|
|
let motionRegion = null;
|
|
if (mode !== 'csi' && videoCapture.isActive) {
|
|
const t0 = performance.now();
|
|
const frame = videoCapture.captureFrame(56, 56);
|
|
if (frame) {
|
|
videoEmb = visualCnn.extract(frame.rgb, frame.width, frame.height);
|
|
motionRegion = videoCapture.detectMotionRegion(56, 56);
|
|
|
|
// Feed motion to CSI simulator for correlated demo data
|
|
// When detected=false, CSI simulator handles through-wall persistence
|
|
csiSimulator.updatePersonState(
|
|
motionRegion.detected ? 1.0 : 0,
|
|
motionRegion.detected ? motionRegion.x + motionRegion.w / 2 : 0.5,
|
|
motionRegion.detected ? motionRegion.y + motionRegion.h / 2 : 0.5,
|
|
frame.motion
|
|
);
|
|
|
|
fusionEngine.updateConfidence(
|
|
frame.brightness, frame.motion,
|
|
0, csiSimulator.isLive || mode === 'dual'
|
|
);
|
|
}
|
|
latency.video = performance.now() - t0;
|
|
}
|
|
|
|
// --- CSI Pipeline ---
|
|
let csiEmb = null;
|
|
if (mode !== 'video') {
|
|
const t0 = performance.now();
|
|
const csiFrame = csiSimulator.nextFrame(elapsed);
|
|
const pseudoImage = csiSimulator.buildPseudoImage(56);
|
|
csiEmb = csiCnn.extract(pseudoImage, 56, 56);
|
|
|
|
fusionEngine.updateConfidence(
|
|
videoCapture.brightnessScore,
|
|
videoCapture.motionScore,
|
|
csiFrame.snr,
|
|
true
|
|
);
|
|
|
|
// Draw CSI heatmap
|
|
const heatmap = csiSimulator.getHeatmapData();
|
|
renderer.drawCsiHeatmap(csiCtx, heatmap, csiCanvas.width, csiCanvas.height);
|
|
|
|
latency.csi = performance.now() - t0;
|
|
}
|
|
|
|
// --- Fusion ---
|
|
const t0f = performance.now();
|
|
const fusedEmb = fusionEngine.fuse(videoEmb, csiEmb, mode);
|
|
latency.fusion = performance.now() - t0f;
|
|
|
|
// --- Pose Decode ---
|
|
// For CSI-only mode, generate a synthetic motion region from CSI energy
|
|
if (mode === 'csi' && (!motionRegion || !motionRegion.detected)) {
|
|
const csiPresence = csiSimulator.personPresence;
|
|
if (csiPresence > 0.1) {
|
|
motionRegion = {
|
|
detected: true,
|
|
x: 0.25, y: 0.15, w: 0.5, h: 0.7,
|
|
coverage: csiPresence,
|
|
motionGrid: null,
|
|
gridCols: 10,
|
|
gridRows: 8
|
|
};
|
|
}
|
|
}
|
|
|
|
// CSI state for through-wall tracking
|
|
const csiState = {
|
|
csiPresence: csiSimulator.personPresence,
|
|
isLive: csiSimulator.isLive
|
|
};
|
|
|
|
const keypoints = poseDecoder.decode(fusedEmb, motionRegion, elapsed, csiState);
|
|
|
|
// --- Render Skeleton ---
|
|
const labelMap = { dual: 'DUAL FUSION', video: 'VIDEO ONLY', csi: 'CSI ONLY' };
|
|
renderer.drawSkeleton(skeletonCtx, keypoints, skeletonCanvas.width, skeletonCanvas.height, {
|
|
minConfidence: confidenceThreshold,
|
|
color: mode === 'csi' ? 'amber' : 'green',
|
|
label: labelMap[mode]
|
|
});
|
|
|
|
// --- Render Embedding Space ---
|
|
const embPoints = fusionEngine.getEmbeddingPoints();
|
|
renderer.drawEmbeddingSpace(embeddingCtx, embPoints, embeddingCanvas.width, embeddingCanvas.height);
|
|
|
|
// --- Update UI ---
|
|
latency.total = performance.now() - totalStart;
|
|
|
|
// FPS
|
|
frameCount++;
|
|
if (timestamp - lastFpsTime > 500) {
|
|
fps = Math.round(frameCount * 1000 / (timestamp - lastFpsTime));
|
|
lastFpsTime = timestamp;
|
|
frameCount = 0;
|
|
fpsDisplay.textContent = `${fps} FPS`;
|
|
}
|
|
|
|
// Fusion bars
|
|
const vc = fusionEngine.videoConfidence;
|
|
const cc = fusionEngine.csiConfidence;
|
|
const fc = fusionEngine.fusedConfidence;
|
|
videoBar.style.width = `${vc * 100}%`;
|
|
csiBar.style.width = `${cc * 100}%`;
|
|
fusedBar.style.width = `${fc * 100}%`;
|
|
videoBarVal.textContent = `${Math.round(vc * 100)}%`;
|
|
csiBarVal.textContent = `${Math.round(cc * 100)}%`;
|
|
fusedBarVal.textContent = `${Math.round(fc * 100)}%`;
|
|
|
|
// Latency
|
|
latVideoEl.textContent = `${latency.video.toFixed(1)}ms`;
|
|
latCsiEl.textContent = `${latency.csi.toFixed(1)}ms`;
|
|
latFusionEl.textContent = `${latency.fusion.toFixed(1)}ms`;
|
|
latTotalEl.textContent = `${latency.total.toFixed(1)}ms`;
|
|
|
|
// Cross-modal similarity
|
|
const sim = fusionEngine.getCrossModalSimilarity();
|
|
crossModalEl.textContent = sim.toFixed(3);
|
|
}
|
|
|
|
// Boot
|
|
document.addEventListener('DOMContentLoaded', init);
|