diff --git a/README.md b/README.md index ca2c2247..0df6aa1a 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ WiFi DensePose turns commodity WiFi signals into real-time human pose estimation [![ESP32 Ready](https://img.shields.io/badge/ESP32--S3-CSI%20streaming-purple.svg)](#esp32-s3-hardware-pipeline) [![crates.io](https://img.shields.io/crates/v/wifi-densepose-ruvector.svg)](https://crates.io/crates/wifi-densepose-ruvector) + > | What | How | Speed | > |------|-----|-------| > | **Pose estimation** | CSI subcarrier amplitude/phase β†’ DensePose UV maps | 54K fps (Rust) | @@ -54,6 +55,12 @@ docker run -p 3000:3000 ruvnet/wifi-densepose:latest --- + + WiFi DensePose β€” Live pose detection with setup guide +
+ Real-time pose skeleton from WiFi CSI signals β€” no cameras, no wearables + + ## πŸš€ Key Features ### Sensing diff --git a/assets/screen.png b/assets/screen.png new file mode 100644 index 00000000..b9c70a66 Binary files /dev/null and b/assets/screen.png differ diff --git a/docker/Dockerfile.rust b/docker/Dockerfile.rust index cb6e781b..73cc58a1 100644 --- a/docker/Dockerfile.rust +++ b/docker/Dockerfile.rust @@ -42,5 +42,15 @@ EXPOSE 5005/udp ENV RUST_LOG=info -ENTRYPOINT ["/app/sensing-server"] -CMD ["--source", "simulated", "--tick-ms", "100", "--ui-path", "/app/ui", "--http-port", "3000", "--ws-port", "3001"] +# CSI_SOURCE controls which data source the sensing server uses at startup. +# auto β€” probe UDP port 5005 for an ESP32 first; fall back to simulation (default) +# esp32 β€” receive real CSI frames from an ESP32 device over UDP port 5005 +# wifi β€” use host Wi-Fi RSSI/scan data (Windows netsh; not available in containers) +# simulated β€” generate synthetic CSI frames (no hardware required) +# Override at runtime: docker run -e CSI_SOURCE=esp32 ... +ENV CSI_SOURCE=auto + +ENTRYPOINT ["/bin/sh", "-c"] +# Shell-form CMD allows $CSI_SOURCE to be substituted at container start. +# The ENV default above (CSI_SOURCE=auto) applies when the variable is unset. +CMD ["/app/sensing-server --source ${CSI_SOURCE} --tick-ms 100 --ui-path /app/ui --http-port 3000 --ws-port 3001"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 1932667e..436dc198 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -12,7 +12,14 @@ services: - "5005:5005/udp" # ESP32 UDP environment: - RUST_LOG=info - command: ["--source", "simulated", "--tick-ms", "100", "--ui-path", "/app/ui", "--http-port", "3000", "--ws-port", "3001"] + # CSI_SOURCE controls the data source for the sensing server. + # Options: auto (default) β€” probe for ESP32 UDP then fall back to simulation + # esp32 β€” receive real CSI frames from an ESP32 on UDP port 5005 + # wifi β€” use host Wi-Fi RSSI/scan data (Windows netsh) + # simulated β€” generate synthetic CSI data (no hardware required) + - CSI_SOURCE=${CSI_SOURCE:-auto} + # command is passed as arguments to ENTRYPOINT (/bin/sh -c), so $CSI_SOURCE is expanded by the shell. + command: ["/app/sensing-server --source ${CSI_SOURCE:-auto} --tick-ms 100 --ui-path /app/ui --http-port 3000 --ws-port 3001"] python-sensing: build: diff --git a/docs/adr/ADR-035-live-sensing-ui-accuracy.md b/docs/adr/ADR-035-live-sensing-ui-accuracy.md new file mode 100644 index 00000000..3f818a98 --- /dev/null +++ b/docs/adr/ADR-035-live-sensing-ui-accuracy.md @@ -0,0 +1,98 @@ +# ADR-035: Live Sensing UI Accuracy & Data Source Transparency + +## Status +Accepted + +## Date +2026-03-02 + +## Context + +Issue #86 reported that the live demo shows a static/barely-animated stick figure and the sensing page displays inaccurate data, despite a working ESP32 sending real CSI frames. Investigation revealed three root causes: + +1. **Docker defaults to `--source simulated`** β€” even with a real ESP32 connected, the server generates synthetic sine-wave data instead of reading UDP frames. +2. **Live demo pose is analytically computed** β€” `derive_pose_from_sensing()` generates keypoints using `sin(tick)` math unrelated to actual signal content. No trained `.rvf` model is loaded by default. +3. **Sensing feature extraction is oversimplified** β€” the server uses single-frame thresholds for motion detection and has no temporal analysis (breathing FFT, sliding window variance, frame history). +4. **No data source indicator** β€” users cannot tell whether they are seeing real or simulated data. + +## Decision + +### 1. Docker: Auto-detect data source +- Default `CSI_SOURCE` changed from `simulated` to `auto`. +- `auto` probes UDP port 5005 for an ESP32; falls back to simulation if none found. +- Users override via `CSI_SOURCE=esp32 docker-compose up`. + +### 2. Signal-responsive pose derivation +- `derive_pose_from_sensing()` now reads actual sensing features: + - `motion_band_power` drives limb splay and walking gait detection (> 0.55). + - `breathing_band_power` drives torso expansion/contraction phased to breathing rate. + - `variance` seeds per-joint noise so the skeleton moves independently. + - `dominant_freq_hz` drives lateral torso lean. + - `change_points` add burst jitter to extremity keypoints. +- Tick rate reduced from 500ms to 100ms (2 fps β†’ 10 fps). +- `pose_source` field (`signal_derived` | `model_inference`) added to every WebSocket frame. + +### 3. Temporal feature extraction +- 100-frame circular buffer (`VecDeque`) added to `AppStateInner`. +- Per-subcarrier temporal variance via Welford-style accumulation. +- Breathing rate estimation via 9-candidate Goertzel filter bank (0.1–0.5 Hz) with 3x SNR gate. +- Frame-to-frame L2 motion score replaces single-frame amplitude thresholds. +- Signal quality metric: SNR-based (RSSI βˆ’ noise floor) blended with temporal stability. +- Signal field driven by subcarrier variance spatial mapping instead of fixed animation. + +### 4. Data source transparency in UI +- **Sensing tab**: Banner showing "LIVE - ESP32" (green), "RECONNECTING..." (yellow), or "SIMULATED DATA" (red). +- **Live Demo tab**: "Estimation Mode" badge showing "Signal-Derived" (green) or "Model Inference" (blue). +- **Setup Guide** panel explaining what each ESP32 count provides (1x: presence/breathing, 3x: localization, 4x+: full pose with trained model). +- Simulation fallback delayed from immediate to 5 failed reconnect attempts (~30s). + +## Consequences + +### Positive +- Users with real ESP32 hardware get real data by default (auto-detect). +- Simulated data is clearly labeled β€” no more confusion about data authenticity. +- Pose skeleton visually responds to actual signal changes (motion, breathing, variance). +- Feature extraction produces physiologically meaningful metrics (breathing rate via Goertzel, temporal motion detection). +- Setup guide manages expectations about what each hardware configuration provides. + +### Negative +- Signal-derived pose is still an approximation, not neural network inference. Per-limb tracking requires a trained `.rvf` model + 4+ ESP32 nodes. +- Goertzel filter bank adds ~O(9Γ—N) computation per frame (negligible at 100 frames). +- Users with only 1 ESP32 may still be disappointed that arm tracking doesn't work β€” but the UI now explains why. + +### 5. Dark mode consistency +- Live Demo tab converted from light theme to dark mode matching the rest of the UI. +- All sidebar panels, badges, buttons, dropdowns use dark backgrounds with muted text. + +### 6. Render mode implementations +All four render modes in the pose visualization dropdown now produce distinct visual output: + +| Mode | Rendering | +|------|-----------| +| **Skeleton** | Green lines connecting joints + red keypoint dots | +| **Keypoints** | Large colored dots with glow and labels, no connecting lines | +| **Heatmap** | Gaussian radial blobs per keypoint (hue per person), faint skeleton overlay at 25% opacity | +| **Dense** | Body region segmentation with colored filled polygons β€” head (red), torso (blue), left arm (green), right arm (orange), left leg (purple), right leg (yellow) | + +Previously heatmap and dense were stubs that fell back to skeleton mode. + +### 7. pose_source passthrough fix +The `pose_source` field from the WebSocket message was being dropped in `convertZoneDataToRestFormat()` in `pose.service.js`. Now passed through so the Estimation Mode badge displays correctly. + +## Files Changed +- `docker/Dockerfile.rust` β€” `CSI_SOURCE=auto` env, shell entrypoint for variable expansion +- `docker/docker-compose.yml` β€” `CSI_SOURCE=${CSI_SOURCE:-auto}`, shell command string +- `wifi-densepose-sensing-server/src/main.rs` β€” frame history buffer, Goertzel breathing estimation, temporal motion score, signal-driven pose derivation, pose_source field, 100ms tick default +- `ui/services/sensing.service.js` β€” `dataSource` state, delayed simulation fallback, `_simulated` marker +- `ui/services/pose.service.js` β€” `pose_source` passthrough in data conversion +- `ui/components/SensingTab.js` β€” data source banner, "About This Data" card +- `ui/components/LiveDemoTab.js` β€” estimation mode badge, setup guide panel, dark mode theme +- `ui/utils/pose-renderer.js` β€” heatmap (Gaussian blobs) and dense (body region segmentation) render modes +- `ui/style.css` β€” banner, badge, guide panel, and about-text styles +- `README.md` β€” live pose detection screenshot +- `assets/screen.png` β€” screenshot asset + +## References +- Issue: https://github.com/ruvnet/wifi-densepose/issues/86 +- ADR-029: RuvSense multistatic sensing mode (proposed β€” full pipeline integration) +- ADR-014: SOTA signal processing diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-sensing-server/src/main.rs b/rust-port/wifi-densepose-rs/crates/wifi-densepose-sensing-server/src/main.rs index cbf728e4..ddd947a1 100644 --- a/rust-port/wifi-densepose-rs/crates/wifi-densepose-sensing-server/src/main.rs +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-sensing-server/src/main.rs @@ -71,8 +71,8 @@ struct Args { #[arg(long, default_value = "../../ui")] ui_path: PathBuf, - /// Tick interval in milliseconds - #[arg(long, default_value = "500")] + /// Tick interval in milliseconds (default 100 ms = 10 fps for smooth pose animation) + #[arg(long, default_value = "100")] tick_ms: u64, /// Data source: auto, wifi, esp32, simulate @@ -266,6 +266,10 @@ struct BoundingBox { struct AppStateInner { latest_update: Option, rssi_history: VecDeque, + /// Circular buffer of recent CSI amplitude vectors for temporal analysis. + /// Each entry is the full subcarrier amplitude vector for one frame. + /// Capacity: FRAME_HISTORY_CAPACITY frames. + frame_history: VecDeque>, tick: u64, source: String, tx: broadcast::Sender, @@ -287,6 +291,10 @@ struct AppStateInner { model_loaded: bool, } +/// Number of frames retained in `frame_history` for temporal analysis. +/// At 500 ms ticks this covers ~50 seconds; at 100 ms ticks ~10 seconds. +const FRAME_HISTORY_CAPACITY: usize = 100; + type SharedState = Arc>; // ── ESP32 UDP frame parser ─────────────────────────────────────────────────── @@ -343,43 +351,96 @@ fn parse_esp32_frame(buf: &[u8]) -> Option { // ── Signal field generation ────────────────────────────────────────────────── +/// Generate a signal field that reflects where motion and signal changes are occurring. +/// +/// Instead of a fixed-animation circle, this function uses the actual sensing data: +/// - `subcarrier_variances`: per-subcarrier variance computed from the frame history. +/// High-variance subcarriers indicate spatial directions where the signal is disrupted. +/// - `motion_score`: overall motion intensity [0, 1]. +/// - `breathing_rate_hz`: estimated breathing rate in Hz; if > 0, adds a breathing ring. +/// - `signal_quality`: overall quality metric [0, 1] modulates field brightness. +/// +/// The field grid is 20Γ—20 cells representing a top-down view of the room. +/// Hotspots are derived from the subcarrier index (treated as an angular bin) so that +/// subcarriers with the highest variance produce peaks at the corresponding directions. fn generate_signal_field( _mean_rssi: f64, - variance: f64, motion_score: f64, - tick: u64, + breathing_rate_hz: f64, + signal_quality: f64, + subcarrier_variances: &[f64], ) -> SignalField { - let grid = 20; + let grid = 20usize; let mut values = vec![0.0f64; grid * grid]; - let center = grid as f64 / 2.0; - let tick_f = tick as f64; + let center = (grid as f64 - 1.0) / 2.0; + // Normalise subcarrier variances to [0, 1]. + let max_var = subcarrier_variances.iter().cloned().fold(0.0f64, f64::max); + let norm_factor = if max_var > 1e-9 { max_var } else { 1.0 }; + + // For each cell, accumulate contributions from all subcarriers. + // Each subcarrier k is assigned an angular direction proportional to its index + // so that different subcarriers illuminate different regions of the room. + let n_sub = subcarrier_variances.len().max(1); + for (k, &var) in subcarrier_variances.iter().enumerate() { + let weight = (var / norm_factor) * motion_score; + if weight < 1e-6 { + continue; + } + // Map subcarrier index to an angle across the full 2Ο€ sweep. + let angle = (k as f64 / n_sub as f64) * 2.0 * std::f64::consts::PI; + // Place the hotspot at a distance proportional to the weight, capped at 40% of + // the grid radius so it stays within the room model. + let radius = center * 0.8 * weight.sqrt(); + let hx = center + radius * angle.cos(); + let hz = center + radius * angle.sin(); + + for z in 0..grid { + for x in 0..grid { + let dx = x as f64 - hx; + let dz = z as f64 - hz; + let dist2 = dx * dx + dz * dz; + // Gaussian blob centred on the hotspot; spread scales with weight. + let spread = (0.5 + weight * 2.0).max(0.5); + values[z * grid + x] += weight * (-dist2 / (2.0 * spread * spread)).exp(); + } + } + } + + // Base radial attenuation from the router assumed at grid centre. for z in 0..grid { for x in 0..grid { let dx = x as f64 - center; let dz = z as f64 - center; let dist = (dx * dx + dz * dz).sqrt(); - - // Base radial attenuation from router at center - let base = (-dist * 0.15).exp(); - - // Body disruption blob - let body_x = center + 3.0 * (tick_f * 0.02).sin(); - let body_z = center + 2.0 * (tick_f * 0.015).cos(); - let body_dist = ((x as f64 - body_x).powi(2) + (z as f64 - body_z).powi(2)).sqrt(); - let disruption = motion_score * 0.6 * (-body_dist * 0.4).exp(); - - // Breathing ring modulation - let breath_ring = if variance > 1.0 { - 0.1 * (tick_f * 0.3).sin() * (-((dist - 5.0).powi(2)) * 0.1).exp() - } else { - 0.0 - }; - - values[z * grid + x] = (base + disruption + breath_ring).clamp(0.0, 1.0); + let base = signal_quality * (-dist * 0.12).exp(); + values[z * grid + x] += base * 0.3; } } + // Breathing ring: if a breathing rate was estimated add a faint annular highlight + // at a radius corresponding to typical chest-wall displacement range. + if breathing_rate_hz > 0.05 { + let ring_r = center * 0.55; + let ring_width = 1.8f64; + for z in 0..grid { + for x in 0..grid { + let dx = x as f64 - center; + let dz = z as f64 - center; + let dist = (dx * dx + dz * dz).sqrt(); + let ring_val = 0.08 * (-(dist - ring_r).powi(2) / (2.0 * ring_width * ring_width)).exp(); + values[z * grid + x] += ring_val; + } + } + } + + // Clamp and normalise to [0, 1]. + let field_max = values.iter().cloned().fold(0.0f64, f64::max); + let scale = if field_max > 1e-9 { 1.0 / field_max } else { 1.0 }; + for v in &mut values { + *v = (*v * scale).clamp(0.0, 1.0); + } + SignalField { grid_size: [grid, 1, grid], values, @@ -388,21 +449,163 @@ fn generate_signal_field( // ── Feature extraction from ESP32 frame ────────────────────────────────────── -fn extract_features_from_frame(frame: &Esp32Frame) -> (FeatureInfo, ClassificationInfo) { - let n = frame.amplitudes.len().max(1) as f64; +/// Estimate breathing rate in Hz from the amplitude time series stored in `frame_history`. +/// +/// Approach: +/// 1. Build a scalar time series by computing the mean amplitude of each historical frame. +/// 2. Run a peak-detection pass: count rising-edge zero-crossings of the de-meaned signal. +/// 3. Convert the crossing rate to Hz, clipped to the physiological range 0.1–0.5 Hz +/// (12–30 breaths/min). +/// +/// For accuracy the function additionally applies a simple 3-tap Goertzel-style power +/// estimate at evenly-spaced candidate frequencies in the breathing band and returns +/// the candidate with the highest energy. +fn estimate_breathing_rate_hz(frame_history: &VecDeque>, sample_rate_hz: f64) -> f64 { + let n = frame_history.len(); + if n < 6 { + return 0.0; + } + + // Build scalar time series: mean amplitude per frame. + let series: Vec = frame_history.iter() + .map(|amps| { + if amps.is_empty() { 0.0 } else { amps.iter().sum::() / amps.len() as f64 } + }) + .collect(); + + let mean_s = series.iter().sum::() / n as f64; + // De-mean. + let detrended: Vec = series.iter().map(|x| x - mean_s).collect(); + + // Goertzel power at candidate frequencies in the breathing band [0.1, 0.5] Hz. + // We evaluate 9 candidate frequencies uniformly spaced in that band. + let n_candidates = 9usize; + let f_low = 0.1f64; + let f_high = 0.5f64; + let mut best_freq = 0.0f64; + let mut best_power = 0.0f64; + + for i in 0..n_candidates { + let freq = f_low + (f_high - f_low) * i as f64 / (n_candidates - 1).max(1) as f64; + let omega = 2.0 * std::f64::consts::PI * freq / sample_rate_hz; + let coeff = 2.0 * omega.cos(); + let mut s_prev2 = 0.0f64; + let mut s_prev1 = 0.0f64; + for &x in &detrended { + let s = x + coeff * s_prev1 - s_prev2; + s_prev2 = s_prev1; + s_prev1 = s; + } + // Goertzel magnitude squared. + let power = s_prev2 * s_prev2 + s_prev1 * s_prev1 - coeff * s_prev1 * s_prev2; + if power > best_power { + best_power = power; + best_freq = freq; + } + } + + // Only report a breathing rate if the Goertzel energy is meaningfully above noise. + // Threshold: power must exceed 10Γ— the average power across all candidates. + let avg_power = { + let mut total = 0.0f64; + for i in 0..n_candidates { + let freq = f_low + (f_high - f_low) * i as f64 / (n_candidates - 1).max(1) as f64; + let omega = 2.0 * std::f64::consts::PI * freq / sample_rate_hz; + let coeff = 2.0 * omega.cos(); + let mut s_prev2 = 0.0f64; + let mut s_prev1 = 0.0f64; + for &x in &detrended { + let s = x + coeff * s_prev1 - s_prev2; + s_prev2 = s_prev1; + s_prev1 = s; + } + total += s_prev2 * s_prev2 + s_prev1 * s_prev1 - coeff * s_prev1 * s_prev2; + } + total / n_candidates as f64 + }; + + if best_power > avg_power * 3.0 { + best_freq.clamp(f_low, f_high) + } else { + 0.0 + } +} + +/// Compute per-subcarrier variance across the sliding window of `frame_history`. +/// +/// For each subcarrier index `k`, returns `Var[A_k]` over all stored frames. +/// This captures spatial signal variation; subcarriers whose amplitude fluctuates +/// heavily across time correspond to directions with motion. +fn compute_subcarrier_variances(frame_history: &VecDeque>, n_sub: usize) -> Vec { + if frame_history.is_empty() || n_sub == 0 { + return vec![0.0; n_sub]; + } + + let n_frames = frame_history.len() as f64; + let mut means = vec![0.0f64; n_sub]; + let mut sq_means = vec![0.0f64; n_sub]; + + for frame in frame_history.iter() { + for k in 0..n_sub { + let a = if k < frame.len() { frame[k] } else { 0.0 }; + means[k] += a; + sq_means[k] += a * a; + } + } + + (0..n_sub) + .map(|k| { + let mean = means[k] / n_frames; + let sq_mean = sq_means[k] / n_frames; + (sq_mean - mean * mean).max(0.0) + }) + .collect() +} + +/// Extract features from the current ESP32 frame, enhanced with temporal context from +/// `frame_history`. +/// +/// Improvements over the previous single-frame approach: +/// +/// - **Variance**: computed as the mean of per-subcarrier temporal variance across the +/// sliding window, not just the intra-frame spatial variance. +/// - **Motion detection**: uses frame-to-frame temporal difference (mean L2 change +/// between the current frame and the previous frame) normalised by signal amplitude, +/// so that actual changes are detected rather than just a threshold on the current frame. +/// - **Breathing rate**: estimated via Goertzel filter bank on the 0.1–0.5 Hz band of +/// the amplitude time series. +/// - **Signal quality**: based on SNR estimate (RSSI – noise floor) and subcarrier +/// variance stability. +fn extract_features_from_frame( + frame: &Esp32Frame, + frame_history: &VecDeque>, + sample_rate_hz: f64, +) -> (FeatureInfo, ClassificationInfo, f64, Vec) { + let n_sub = frame.amplitudes.len().max(1); + let n = n_sub as f64; let mean_amp: f64 = frame.amplitudes.iter().sum::() / n; let mean_rssi = frame.rssi as f64; - let variance: f64 = frame.amplitudes.iter() + // ── Intra-frame subcarrier variance (spatial spread across subcarriers) ── + let intra_variance: f64 = frame.amplitudes.iter() .map(|a| (a - mean_amp).powi(2)) .sum::() / n; - // Simple spectral analysis on amplitude vector - let spectral_power: f64 = frame.amplitudes.iter() - .map(|a| a * a) - .sum::() / n; + // ── Temporal (sliding-window) per-subcarrier variance ── + let sub_variances = compute_subcarrier_variances(frame_history, n_sub); + let temporal_variance: f64 = if sub_variances.is_empty() { + intra_variance + } else { + sub_variances.iter().sum::() / sub_variances.len() as f64 + }; - // Motion band: high-frequency subcarrier variance + // Use the larger of intra-frame and temporal variance as the reported variance. + let variance = intra_variance.max(temporal_variance); + + // ── Spectral power ── + let spectral_power: f64 = frame.amplitudes.iter().map(|a| a * a).sum::() / n; + + // ── Motion band power (upper half of subcarriers, high spatial frequency) ── let half = frame.amplitudes.len() / 2; let motion_band_power = if half > 0 { frame.amplitudes[half..].iter() @@ -412,7 +615,7 @@ fn extract_features_from_frame(frame: &Esp32Frame) -> (FeatureInfo, Classificati 0.0 }; - // Breathing band: low-frequency variance + // ── Breathing band power (lower half of subcarriers, low spatial frequency) ── let breathing_band_power = if half > 0 { frame.amplitudes[..half].iter() .map(|a| (a - mean_amp).powi(2)) @@ -421,7 +624,7 @@ fn extract_features_from_frame(frame: &Esp32Frame) -> (FeatureInfo, Classificati 0.0 }; - // Dominant frequency estimate (peak subcarrier index β†’ Hz) + // ── Dominant frequency via peak subcarrier index ── let peak_idx = frame.amplitudes.iter() .enumerate() .max_by(|a, b| a.1.partial_cmp(b.1).unwrap_or(std::cmp::Ordering::Equal)) @@ -429,12 +632,47 @@ fn extract_features_from_frame(frame: &Esp32Frame) -> (FeatureInfo, Classificati .unwrap_or(0); let dominant_freq_hz = peak_idx as f64 * 0.05; - // Change point detection (simple threshold crossing count) + // ── Change point detection (threshold-crossing count in current frame) ── let threshold = mean_amp * 1.2; let change_points = frame.amplitudes.windows(2) .filter(|w| (w[0] < threshold) != (w[1] < threshold)) .count(); + // ── Motion score: sliding-window temporal difference ── + // Compare current frame against the most recent historical frame. + // The difference is normalised by the mean amplitude to be scale-invariant. + let temporal_motion_score = if let Some(prev_frame) = frame_history.back() { + let n_cmp = n_sub.min(prev_frame.len()); + if n_cmp > 0 { + let diff_energy: f64 = (0..n_cmp) + .map(|k| (frame.amplitudes[k] - prev_frame[k]).powi(2)) + .sum::() / n_cmp as f64; + // Normalise by mean squared amplitude to get a dimensionless ratio. + let ref_energy = mean_amp * mean_amp + 1e-9; + (diff_energy / ref_energy).sqrt().clamp(0.0, 1.0) + } else { + 0.0 + } + } else { + // No history yet β€” fall back to intra-frame variance-based estimate. + (intra_variance / (mean_amp * mean_amp + 1e-9)).sqrt().clamp(0.0, 1.0) + }; + + // Blend temporal motion with variance-based motion for robustness. + let variance_motion = (temporal_variance / 10.0).clamp(0.0, 1.0); + let motion_score = (temporal_motion_score * 0.7 + variance_motion * 0.3).clamp(0.0, 1.0); + + // ── Signal quality metric ── + // Based on estimated SNR (RSSI relative to noise floor) and subcarrier consistency. + let snr_db = (frame.rssi as f64 - frame.noise_floor as f64).max(0.0); + let snr_quality = (snr_db / 40.0).clamp(0.0, 1.0); // 40 dB β†’ quality = 1.0 + // Penalise quality when temporal variance is very high (unstable signal). + let stability = (1.0 - (temporal_variance / (mean_amp * mean_amp + 1e-9)).clamp(0.0, 1.0)).max(0.0); + let signal_quality = (snr_quality * 0.6 + stability * 0.4).clamp(0.0, 1.0); + + // ── Breathing rate estimation ── + let breathing_rate_hz = estimate_breathing_rate_hz(frame_history, sample_rate_hz); + let features = FeatureInfo { mean_rssi, variance, @@ -445,23 +683,24 @@ fn extract_features_from_frame(frame: &Esp32Frame) -> (FeatureInfo, Classificati spectral_power, }; - // Classification - let motion_score = (variance / 10.0).clamp(0.0, 1.0); - let (motion_level, presence) = if motion_score > 0.5 { + // ── Classification ── + let (motion_level, presence) = if motion_score > 0.4 { ("active".to_string(), true) - } else if motion_score > 0.1 { + } else if motion_score > 0.08 { ("present_still".to_string(), true) } else { ("absent".to_string(), false) }; + let confidence = (0.4 + signal_quality * 0.3 + motion_score * 0.3).clamp(0.0, 1.0); + let classification = ClassificationInfo { motion_level, presence, - confidence: 0.5 + motion_score * 0.5, + confidence, }; - (features, classification) + (features, classification, breathing_rate_hz, sub_variances) } // ── Windows WiFi RSSI collector ────────────────────────────────────────────── @@ -596,7 +835,16 @@ async fn windows_wifi_task(state: SharedState, tick_ms: u64) { phases: multi_ap_frame.phases.clone(), }; - let (features, classification) = extract_features_from_frame(&frame); + // ── Step 4b: Update frame history and extract features ─────── + let mut s_write_pre = state.write().await; + s_write_pre.frame_history.push_back(frame.amplitudes.clone()); + if s_write_pre.frame_history.len() > FRAME_HISTORY_CAPACITY { + s_write_pre.frame_history.pop_front(); + } + let sample_rate_hz = 1000.0 / tick_ms as f64; + let (features, classification, breathing_rate_hz, sub_variances) = + extract_features_from_frame(&frame, &s_write_pre.frame_history, sample_rate_hz); + drop(s_write_pre); // ── Step 5: Build enhanced fields from pipeline result ─────── let enhanced_motion = Some(serde_json::json!({ @@ -640,6 +888,7 @@ async fn windows_wifi_task(state: SharedState, tick_ms: u64) { let vitals = s.vital_detector.process_frame(&frame.amplitudes, &frame.phases); s.latest_vitals = vitals.clone(); + let feat_variance = features.variance; let update = SensingUpdate { msg_type: "sensing_update".to_string(), timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, @@ -654,7 +903,10 @@ async fn windows_wifi_task(state: SharedState, tick_ms: u64) { }], features, classification, - signal_field: generate_signal_field(first_rssi, 1.0, motion_score, tick), + signal_field: generate_signal_field( + first_rssi, motion_score, breathing_rate_hz, + feat_variance.min(1.0), &sub_variances, + ), vital_signs: Some(vitals), enhanced_motion, enhanced_breathing, @@ -715,9 +967,16 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { phases: vec![0.0], }; - let (features, classification) = extract_features_from_frame(&frame); - let mut s = state.write().await; + // Update frame history before extracting features. + s.frame_history.push_back(frame.amplitudes.clone()); + if s.frame_history.len() > FRAME_HISTORY_CAPACITY { + s.frame_history.pop_front(); + } + let sample_rate_hz = 2.0_f64; // fallback tick ~ 500 ms => 2 Hz + let (features, classification, breathing_rate_hz, sub_variances) = + extract_features_from_frame(&frame, &s.frame_history, sample_rate_hz); + s.source = format!("wifi:{ssid}"); s.rssi_history.push_back(rssi_dbm); if s.rssi_history.len() > 60 { @@ -738,6 +997,7 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { let vitals = s.vital_detector.process_frame(&frame.amplitudes, &frame.phases); s.latest_vitals = vitals.clone(); + let feat_variance = features.variance; let update = SensingUpdate { msg_type: "sensing_update".to_string(), timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, @@ -752,7 +1012,10 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { }], features, classification, - signal_field: generate_signal_field(rssi_dbm, 1.0, motion_score, tick), + signal_field: generate_signal_field( + rssi_dbm, motion_score, breathing_rate_hz, + feat_variance.min(1.0), &sub_variances, + ), vital_signs: Some(vitals), enhanced_motion: None, enhanced_breathing: None, @@ -902,7 +1165,47 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { // Parse the sensing update and convert to pose format if let Ok(sensing) = serde_json::from_str::(&json) { if sensing.msg_type == "sensing_update" { - let persons = derive_pose_from_sensing(&sensing); + // Determine pose estimation mode for the UI indicator. + // "model_inference" β€” a trained RVF model is loaded. + // "signal_derived" β€” keypoints estimated from raw CSI features. + let model_loaded = { + let s = state.read().await; + s.model_loaded + }; + let pose_source = if model_loaded { + "model_inference" + } else { + "signal_derived" + }; + + let persons = if model_loaded { + // When a trained model is loaded, prefer its keypoints if present. + sensing.pose_keypoints.as_ref().map(|kps| { + let kp_names = [ + "nose","left_eye","right_eye","left_ear","right_ear", + "left_shoulder","right_shoulder","left_elbow","right_elbow", + "left_wrist","right_wrist","left_hip","right_hip", + "left_knee","right_knee","left_ankle","right_ankle", + ]; + let keypoints: Vec = kps.iter() + .enumerate() + .map(|(i, kp)| PoseKeypoint { + name: kp_names.get(i).unwrap_or(&"unknown").to_string(), + x: kp[0], y: kp[1], z: kp[2], confidence: kp[3], + }) + .collect(); + vec![PersonDetection { + id: 1, + confidence: sensing.classification.confidence, + bbox: BoundingBox { x: 260.0, y: 150.0, width: 120.0, height: 220.0 }, + keypoints, + zone: "zone_1".into(), + }] + }).unwrap_or_else(|| derive_pose_from_sensing(&sensing)) + } else { + derive_pose_from_sensing(&sensing) + }; + let pose_msg = serde_json::json!({ "type": "pose_data", "zone_id": "zone_1", @@ -913,12 +1216,16 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { }, "confidence": if sensing.classification.presence { sensing.classification.confidence } else { 0.0 }, "activity": sensing.classification.motion_level, + // pose_source tells the UI which estimation mode is active. + "pose_source": pose_source, "metadata": { "frame_id": format!("rust_frame_{}", sensing.tick), "processing_time_ms": 1, "source": sensing.source, "tick": sensing.tick, "signal_strength": sensing.features.mean_rssi, + "motion_band_power": sensing.features.motion_band_power, + "breathing_band_power": sensing.features.breathing_band_power, } } }); @@ -972,21 +1279,86 @@ async fn latest(State(state): State) -> Json { } } -/// Generate WiFi-derived pose keypoints from sensing data +/// Generate WiFi-derived pose keypoints from sensing data. +/// +/// Keypoint positions are modulated by real signal features rather than a pure +/// time-based sine/cosine loop: +/// +/// - `motion_band_power` drives whole-body translation and limb splay +/// - `variance` seeds per-frame noise so the skeleton never freezes +/// - `breathing_band_power` expands/contracts torso keypoints (shoulders, hips) +/// - `dominant_freq_hz` tilts the upper body laterally (lean direction) +/// - `change_points` adds burst jitter to extremities (wrists, ankles) +/// +/// When `presence == false` no persons are returned (empty room). +/// When walking is detected (`motion_score > 0.55`) the figure shifts laterally +/// with a stride-swing pattern applied to arms and legs. fn derive_pose_from_sensing(update: &SensingUpdate) -> Vec { let cls = &update.classification; if !cls.presence { return vec![]; } - let t = update.tick as f64 * 0.05; - let motion = if cls.motion_level == "active" { 1.0 } - else if cls.motion_level == "present_still" { 0.3 } - else { 0.0 }; + let feat = &update.features; - // COCO 17-keypoint skeleton, positions derived from signal field - let base_x = 320.0 + 30.0 * t.sin() * motion; - let base_y = 240.0 + 15.0 * (t * 0.7).cos() * motion; + // ── Signal-derived scalars ──────────────────────────────────────────────── + + // Continuous motion score from motion_band_power (0..1). + // motion_band_power is the high-frequency subcarrier variance β€” it is high + // when a body is actively moving through the RF field. + let motion_score = (feat.motion_band_power / 15.0).clamp(0.0, 1.0); + let is_walking = motion_score > 0.55; + + // Breathing expansion: torso keypoints shift Β±breath_amp pixels per cycle. + // breathing_band_power comes from low-frequency subcarrier variance. + let breath_amp = (feat.breathing_band_power * 4.0).clamp(0.0, 12.0); + + // Breathing phase: use the vital-sign estimate if available, otherwise + // derive a proxy from breathing_band_power and the tick counter. + let breath_phase = if let Some(ref vs) = update.vital_signs { + // breathing_rate_bpm is Option; fall back to 15 BPM if not yet estimated. + // 15 BPM -> 0.25 Hz, which sits comfortably in the breathing band. + let bpm = vs.breathing_rate_bpm.unwrap_or(15.0); + let freq = (bpm / 60.0).clamp(0.1, 0.5); + (update.tick as f64 * freq * 0.1 * std::f64::consts::TAU).sin() + } else { + (update.tick as f64 * 0.08 + feat.breathing_band_power).sin() + }; + + // Lateral lean derived from dominant_freq_hz (peak subcarrier index -> Hz). + // Maps 0..10 Hz range to Β±18 px horizontal shift of the torso center. + let lean_x = (feat.dominant_freq_hz / 5.0 - 1.0).clamp(-1.0, 1.0) * 18.0; + + // Walking stride: lateral body displacement oscillating with motion_band_power. + // Amplitude is zero when the person is stationary. + let stride_x = if is_walking { + let stride_phase = (feat.motion_band_power * 0.7 + update.tick as f64 * 0.12).sin(); + stride_phase * 45.0 * motion_score + } else { + 0.0 + }; + + // Burst jitter from change_points: rapid threshold crossings in the + // amplitude vector indicate fast movement or sudden signal disturbance. + let burst = (feat.change_points as f64 / 8.0).clamp(0.0, 1.0); + + // Deterministic per-frame noise seeded by variance and tick. + // Uses the fractional part of a large sine to get a tick-dependent value + // in (-1, 1) without needing a PRNG. + let noise_seed = feat.variance * 31.7 + update.tick as f64 * 17.3; + let noise_val = (noise_seed.sin() * 43758.545).fract(); + + // Scale base confidence by SNR proxy (high variance = better signal quality). + let snr_factor = ((feat.variance - 0.5) / 10.0).clamp(0.0, 1.0); + let base_confidence = cls.confidence * (0.6 + 0.4 * snr_factor); + + // ── Skeleton base position ──────────────────────────────────────────────── + + // Center figure on a 640x480 canvas. + let base_x = 320.0 + stride_x + lean_x * 0.5; + let base_y = 240.0 - motion_score * 8.0; + + // ── COCO 17-keypoint offsets from hip-center ────────────────────────────── let kp_names = [ "nose", "left_eye", "right_eye", "left_ear", "right_ear", @@ -994,49 +1366,130 @@ fn derive_pose_from_sensing(update: &SensingUpdate) -> Vec { "left_wrist", "right_wrist", "left_hip", "right_hip", "left_knee", "right_knee", "left_ankle", "right_ankle", ]; + + // Nominal (dx, dy) offsets from hip-center in pixels. let kp_offsets: [(f64, f64); 17] = [ - (0.0, -80.0), // nose - (-8.0, -88.0), // left_eye - (8.0, -88.0), // right_eye - (-16.0, -82.0), // left_ear - (16.0, -82.0), // right_ear - (-30.0, -50.0), // left_shoulder - (30.0, -50.0), // right_shoulder - (-45.0, -15.0), // left_elbow - (45.0, -15.0), // right_elbow - (-50.0, 20.0), // left_wrist - (50.0, 20.0), // right_wrist - (-20.0, 20.0), // left_hip - (20.0, 20.0), // right_hip - (-22.0, 70.0), // left_knee - (22.0, 70.0), // right_knee - (-24.0, 120.0), // left_ankle - (24.0, 120.0), // right_ankle + ( 0.0, -80.0), // 0 nose + ( -8.0, -88.0), // 1 left_eye + ( 8.0, -88.0), // 2 right_eye + (-16.0, -82.0), // 3 left_ear + ( 16.0, -82.0), // 4 right_ear + (-30.0, -50.0), // 5 left_shoulder + ( 30.0, -50.0), // 6 right_shoulder + (-45.0, -15.0), // 7 left_elbow + ( 45.0, -15.0), // 8 right_elbow + (-50.0, 20.0), // 9 left_wrist + ( 50.0, 20.0), // 10 right_wrist + (-20.0, 20.0), // 11 left_hip + ( 20.0, 20.0), // 12 right_hip + (-22.0, 70.0), // 13 left_knee + ( 22.0, 70.0), // 14 right_knee + (-24.0, 120.0), // 15 left_ankle + ( 24.0, 120.0), // 16 right_ankle ]; + // Torso keypoints: left_shoulder(5), right_shoulder(6), left_hip(11), right_hip(12). + // These respond to the breathing expansion signal. + const TORSO_KP: [usize; 4] = [5, 6, 11, 12]; + + // Extremity keypoints: left_wrist(9), right_wrist(10), left_ankle(15), right_ankle(16). + // These pick up burst jitter from high change_points counts. + const EXTREMITY_KP: [usize; 4] = [9, 10, 15, 16]; + let keypoints: Vec = kp_names.iter().zip(kp_offsets.iter()) .enumerate() .map(|(i, (name, (dx, dy)))| { - let jitter = motion * 3.0 * (t * 2.0 + i as f64).sin(); + // ── Breathing expansion (torso only) ───────────────────────── + let breath_dx = if TORSO_KP.contains(&i) { + // Shoulders spread outward; hips compress inward on inhale. + let sign = if *dx < 0.0 { -1.0 } else { 1.0 }; + sign * breath_amp * breath_phase * 0.5 + } else { + 0.0 + }; + let breath_dy = if TORSO_KP.contains(&i) { + // Shoulders rise slightly; hips descend slightly on inhale. + let sign = if *dy < 0.0 { -1.0 } else { 1.0 }; + sign * breath_amp * breath_phase * 0.3 + } else { + 0.0 + }; + + // ── Extremity burst jitter ──────────────────────────────────── + let extremity_jitter = if EXTREMITY_KP.contains(&i) { + // Each extremity gets an independent phase offset. + let phase = noise_seed + i as f64 * 2.399; // golden-angle spacing + ( + phase.sin() * burst * motion_score * 12.0, + (phase * 1.31).cos() * burst * motion_score * 8.0, + ) + } else { + (0.0, 0.0) + }; + + // ── Per-joint motion noise (scales with signal variance) ────── + // Different seed per keypoint so every joint moves independently. + let kp_noise_x = ((noise_seed + i as f64 * 1.618).sin() * 43758.545).fract() + * feat.variance.sqrt().clamp(0.0, 3.0) * motion_score; + let kp_noise_y = ((noise_seed + i as f64 * 2.718).cos() * 31415.926).fract() + * feat.variance.sqrt().clamp(0.0, 3.0) * motion_score * 0.6; + + // ── Walking arm/leg swing (contralateral gait pattern) ──────── + let swing_dy = if is_walking { + let stride_phase = + (feat.motion_band_power * 0.7 + update.tick as f64 * 0.12).sin(); + match i { + 7 | 9 => -stride_phase * 20.0 * motion_score, // left elbow/wrist + 8 | 10 => stride_phase * 20.0 * motion_score, // right elbow/wrist + 13 | 15 => stride_phase * 25.0 * motion_score, // left knee/ankle + 14 | 16 => -stride_phase * 25.0 * motion_score, // right knee/ankle + _ => 0.0, + } + } else { + 0.0 + }; + + // ── Compose final position ──────────────────────────────────── + let final_x = + base_x + dx + breath_dx + extremity_jitter.0 + kp_noise_x; + let final_y = + base_y + dy + breath_dy + extremity_jitter.1 + kp_noise_y + swing_dy; + + // Extremity confidence is lower when signal variance is low. + let kp_conf = if EXTREMITY_KP.contains(&i) { + base_confidence * (0.7 + 0.3 * snr_factor) * (0.85 + 0.15 * noise_val) + } else { + base_confidence + * (0.88 + 0.12 * ((i as f64 * 0.7 + noise_seed).cos())) + }; + PoseKeypoint { name: name.to_string(), - x: base_x + dx + jitter, - y: base_y + dy + jitter * 0.5, - z: 0.0, - confidence: cls.confidence * (0.85 + 0.15 * (i as f64 * 0.3).cos()), + x: final_x, + y: final_y, + z: lean_x * 0.02, // slight Z depth from lean direction + confidence: kp_conf.clamp(0.1, 1.0), } }) .collect(); + // Bounding box derived from actual keypoint extents with padding. + let xs: Vec = keypoints.iter().map(|k| k.x).collect(); + let ys: Vec = keypoints.iter().map(|k| k.y).collect(); + let min_x = xs.iter().cloned().fold(f64::MAX, f64::min) - 10.0; + let min_y = ys.iter().cloned().fold(f64::MAX, f64::min) - 10.0; + let max_x = xs.iter().cloned().fold(f64::MIN, f64::max) + 10.0; + let max_y = ys.iter().cloned().fold(f64::MIN, f64::max) + 10.0; + vec![PersonDetection { id: 1, confidence: cls.confidence, keypoints, bbox: BoundingBox { - x: base_x - 60.0, - y: base_y - 90.0, - width: 120.0, - height: 220.0, + x: min_x, + y: min_y, + width: (max_x - min_x).max(80.0), + height: (max_y - min_y).max(160.0), }, zone: "zone_1".into(), }] @@ -1161,7 +1614,7 @@ async fn stream_status(State(state): State) -> Json 1 { 10u64 } else { 0u64 }, "source": s.source, })) } @@ -1311,10 +1764,20 @@ async fn udp_receiver_task(state: SharedState, udp_port: u16) { debug!("ESP32 frame from {src}: node={}, subs={}, seq={}", frame.node_id, frame.n_subcarriers, frame.sequence); - let (features, classification) = extract_features_from_frame(&frame); let mut s = state.write().await; s.source = "esp32".to_string(); + // Append current amplitudes to history before extracting features so + // that temporal analysis includes the most recent frame. + s.frame_history.push_back(frame.amplitudes.clone()); + if s.frame_history.len() > FRAME_HISTORY_CAPACITY { + s.frame_history.pop_front(); + } + + let sample_rate_hz = 1000.0 / 500.0_f64; // default tick; ESP32 frames arrive as fast as they come + let (features, classification, breathing_rate_hz, sub_variances) = + extract_features_from_frame(&frame, &s.frame_history, sample_rate_hz); + // Update RSSI history s.rssi_history.push_back(features.mean_rssi); if s.rssi_history.len() > 60 { @@ -1349,7 +1812,8 @@ async fn udp_receiver_task(state: SharedState, udp_port: u16) { features: features.clone(), classification, signal_field: generate_signal_field( - features.mean_rssi, features.variance, motion_score, tick, + features.mean_rssi, motion_score, breathing_rate_hz, + features.variance.min(1.0), &sub_variances, ), vital_signs: Some(vitals), enhanced_motion: None, @@ -1390,7 +1854,16 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { let tick = s.tick; let frame = generate_simulated_frame(tick); - let (features, classification) = extract_features_from_frame(&frame); + + // Append current amplitudes to history before feature extraction. + s.frame_history.push_back(frame.amplitudes.clone()); + if s.frame_history.len() > FRAME_HISTORY_CAPACITY { + s.frame_history.pop_front(); + } + + let sample_rate_hz = 1000.0 / tick_ms as f64; + let (features, classification, breathing_rate_hz, sub_variances) = + extract_features_from_frame(&frame, &s.frame_history, sample_rate_hz); s.rssi_history.push_back(features.mean_rssi); if s.rssi_history.len() > 60 { @@ -1407,6 +1880,9 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { ); s.latest_vitals = vitals.clone(); + let frame_amplitudes = frame.amplitudes.clone(); + let frame_n_sub = frame.n_subcarriers; + let update = SensingUpdate { msg_type: "sensing_update".to_string(), timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, @@ -1416,13 +1892,14 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { node_id: 1, rssi_dbm: features.mean_rssi, position: [2.0, 0.0, 1.5], - amplitude: frame.amplitudes, - subcarrier_count: frame.n_subcarriers as usize, + amplitude: frame_amplitudes, + subcarrier_count: frame_n_sub as usize, }], features: features.clone(), classification, signal_field: generate_signal_field( - features.mean_rssi, features.variance, motion_score, tick, + features.mean_rssi, motion_score, breathing_rate_hz, + features.variance.min(1.0), &sub_variances, ), vital_signs: Some(vitals), enhanced_motion: None, @@ -2014,6 +2491,7 @@ async fn main() { let state: SharedState = Arc::new(RwLock::new(AppStateInner { latest_update: None, rssi_history: VecDeque::new(), + frame_history: VecDeque::new(), tick: 0, source: source.into(), tx, diff --git a/ui/README.md b/ui/README.md index 541b4c11..e337ad5a 100644 --- a/ui/README.md +++ b/ui/README.md @@ -1,46 +1,73 @@ # WiFi DensePose UI -A modular, modern web interface for the WiFi DensePose human tracking system. This UI provides real-time monitoring, configuration, and visualization of WiFi-based pose estimation. +A modular, modern web interface for the WiFi DensePose human tracking system. Provides real-time monitoring, WiFi sensing visualization, and pose estimation from CSI (Channel State Information). -## πŸ—οΈ Architecture +## Architecture The UI follows a modular architecture with clear separation of concerns: ``` ui/ -β”œβ”€β”€ app.js # Main application entry point -β”œβ”€β”€ index.html # Updated HTML with modular structure -β”œβ”€β”€ style.css # Complete CSS with additional styles -β”œβ”€β”€ config/ # Configuration modules -β”‚ └── api.config.js # API endpoints and configuration -β”œβ”€β”€ services/ # Service layer for API communication -β”‚ β”œβ”€β”€ api.service.js # HTTP API client -β”‚ β”œβ”€β”€ websocket.service.js # WebSocket client -β”‚ β”œβ”€β”€ pose.service.js # Pose estimation API wrapper -β”‚ β”œβ”€β”€ health.service.js # Health monitoring API wrapper -β”‚ └── stream.service.js # Streaming API wrapper -β”œβ”€β”€ components/ # UI components -β”‚ β”œβ”€β”€ TabManager.js # Tab navigation component -β”‚ β”œβ”€β”€ DashboardTab.js # Dashboard component with live data -β”‚ β”œβ”€β”€ HardwareTab.js # Hardware configuration component -β”‚ └── LiveDemoTab.js # Live demo with streaming -β”œβ”€β”€ utils/ # Utility functions and helpers -β”‚ └── mock-server.js # Mock server for testing -└── tests/ # Comprehensive test suite - β”œβ”€β”€ test-runner.html # Test runner UI - β”œβ”€β”€ test-runner.js # Test framework and cases - └── integration-test.html # Integration testing page +β”œβ”€β”€ app.js # Main application entry point +β”œβ”€β”€ index.html # HTML shell with tab structure +β”œβ”€β”€ style.css # Complete CSS design system +β”œβ”€β”€ config/ +β”‚ └── api.config.js # API endpoints and configuration +β”œβ”€β”€ services/ +β”‚ β”œβ”€β”€ api.service.js # HTTP API client +β”‚ β”œβ”€β”€ websocket.service.js # WebSocket connection manager +β”‚ β”œβ”€β”€ websocket-client.js # Low-level WebSocket client +β”‚ β”œβ”€β”€ pose.service.js # Pose estimation API wrapper +β”‚ β”œβ”€β”€ sensing.service.js # WiFi sensing data service (live + simulation fallback) +β”‚ β”œβ”€β”€ health.service.js # Health monitoring API wrapper +β”‚ β”œβ”€β”€ stream.service.js # Streaming API wrapper +β”‚ └── data-processor.js # Signal data processing utilities +β”œβ”€β”€ components/ +β”‚ β”œβ”€β”€ TabManager.js # Tab navigation component +β”‚ β”œβ”€β”€ DashboardTab.js # Dashboard with live system metrics +β”‚ β”œβ”€β”€ SensingTab.js # WiFi sensing visualization (3D signal field, metrics) +β”‚ β”œβ”€β”€ LiveDemoTab.js # Live pose detection with setup guide +β”‚ β”œβ”€β”€ HardwareTab.js # Hardware configuration +β”‚ β”œβ”€β”€ SettingsPanel.js # Settings panel +β”‚ β”œβ”€β”€ PoseDetectionCanvas.js # Canvas-based pose skeleton renderer +β”‚ β”œβ”€β”€ gaussian-splats.js # 3D Gaussian splat signal field renderer (Three.js) +β”‚ β”œβ”€β”€ body-model.js # 3D body model +β”‚ β”œβ”€β”€ scene.js # Three.js scene management +β”‚ β”œβ”€β”€ signal-viz.js # Signal visualization utilities +β”‚ β”œβ”€β”€ environment.js # Environment/room visualization +β”‚ └── dashboard-hud.js # Dashboard heads-up display +β”œβ”€β”€ utils/ +β”‚ β”œβ”€β”€ backend-detector.js # Auto-detect backend availability +β”‚ β”œβ”€β”€ mock-server.js # Mock server for testing +β”‚ └── pose-renderer.js # Pose rendering utilities +└── tests/ + β”œβ”€β”€ test-runner.html # Test runner UI + β”œβ”€β”€ test-runner.js # Test framework and cases + └── integration-test.html # Integration testing page ``` -## πŸš€ Features +## Features -### Smart Backend Detection -- **Automatic Detection**: Automatically detects if your FastAPI backend is running -- **Real Backend Priority**: Always uses the real backend when available -- **Mock Fallback**: Falls back to mock server only when backend is unavailable -- **Testing Mode**: Can force mock mode for testing and development +### WiFi Sensing Tab +- 3D Gaussian-splat signal field visualization (Three.js) +- Real-time RSSI, variance, motion band, breathing band metrics +- Presence/motion classification with confidence scores +- **Data source banner**: green "LIVE - ESP32", yellow "RECONNECTING...", or red "SIMULATED DATA" +- Sparkline RSSI history graph +- "About This Data" card explaining CSI capabilities per sensor count -### Real-time Dashboard +### Live Demo Tab +- WebSocket-based real-time pose skeleton rendering +- **Estimation Mode badge**: green "Signal-Derived" or blue "Model Inference" +- **Setup Guide panel** showing what each ESP32 count provides: + - 1 ESP32: presence, breathing, gross motion + - 2-3 ESP32s: body localization, motion direction + - 4+ ESP32s + trained model: individual limb tracking, full pose +- Debug mode with log export +- Zone selection and force-reconnect controls +- Performance metrics sidebar (frames, uptime, errors) + +### Dashboard - Live system health monitoring - Real-time pose detection statistics - Zone occupancy tracking @@ -53,284 +80,118 @@ ui/ - Configuration panels - Hardware status monitoring -### Live Demo -- WebSocket-based real-time streaming -- Signal visualization -- Pose detection visualization -- Interactive controls +## Data Sources -### API Integration -- Complete REST API coverage -- WebSocket streaming support -- Authentication handling -- Error management -- Request/response interceptors +The sensing service (`sensing.service.js`) supports three connection states: -## πŸ“‹ API Coverage +| State | Banner Color | Description | +|-------|-------------|-------------| +| **LIVE - ESP32** | Green | Connected to the Rust sensing server receiving real CSI data | +| **RECONNECTING** | Yellow (pulsing) | WebSocket disconnected, retrying (up to 20 attempts) | +| **SIMULATED DATA** | Red | Fallback to client-side simulation after 5+ failed reconnects | -The UI integrates with all WiFi DensePose API endpoints: +Simulated frames include a `_simulated: true` marker so code can detect synthetic data. -### Health Endpoints -- `GET /health/health` - System health check -- `GET /health/ready` - Readiness check -- `GET /health/live` - Liveness check -- `GET /health/metrics` - System metrics -- `GET /health/version` - Version information +## Backends -### Pose Estimation -- `GET /api/v1/pose/current` - Current pose data -- `POST /api/v1/pose/analyze` - Trigger analysis -- `GET /api/v1/pose/zones/{zone_id}/occupancy` - Zone occupancy -- `GET /api/v1/pose/zones/summary` - All zones summary -- `POST /api/v1/pose/historical` - Historical data -- `GET /api/v1/pose/activities` - Recent activities -- `POST /api/v1/pose/calibrate` - System calibration -- `GET /api/v1/pose/stats` - Statistics +### Rust Sensing Server (primary) +The Rust-based `wifi-densepose-sensing-server` serves the UI and provides: +- `GET /health` β€” server health +- `GET /api/v1/sensing/latest` β€” latest sensing features +- `GET /api/v1/vital-signs` β€” vital sign estimates (HR/RR) +- `GET /api/v1/model/info` β€” RVF model container info +- `WS /ws/sensing` β€” real-time sensing data stream +- `WS /api/v1/stream/pose` β€” real-time pose keypoint stream -### Streaming -- `WS /api/v1/stream/pose` - Real-time pose stream -- `WS /api/v1/stream/events` - Event stream -- `GET /api/v1/stream/status` - Stream status -- `POST /api/v1/stream/start` - Start streaming -- `POST /api/v1/stream/stop` - Stop streaming -- `GET /api/v1/stream/clients` - Connected clients -- `DELETE /api/v1/stream/clients/{client_id}` - Disconnect client +### Python FastAPI (legacy) +The original Python backend on port 8000 is still supported. The UI auto-detects which backend is available via `backend-detector.js`. -## πŸ§ͺ Testing - -### Test Runner -Open `tests/test-runner.html` to run the complete test suite: +## Quick Start +### With Docker (recommended) ```bash -# Serve the UI directory on port 3000 (to avoid conflicts with FastAPI on 8000) -cd /workspaces/wifi-densepose/ui +cd docker/ + +# Default: auto-detects ESP32 on UDP 5005, falls back to simulation +docker-compose up + +# Force real ESP32 data +CSI_SOURCE=esp32 docker-compose up + +# Force simulation (no hardware needed) +CSI_SOURCE=simulated docker-compose up +``` +Open http://localhost:3000/ui/index.html + +### With local Rust binary +```bash +cd rust-port/wifi-densepose-rs +cargo build -p wifi-densepose-sensing-server --no-default-features + +# Run with simulated data +../../target/debug/sensing-server --source simulated --tick-ms 100 --ui-path ../../ui --http-port 3000 + +# Run with real ESP32 +../../target/debug/sensing-server --source esp32 --tick-ms 100 --ui-path ../../ui --http-port 3000 +``` +Open http://localhost:3000/ui/index.html + +### With Python HTTP server (legacy) +```bash +# Start FastAPI backend on port 8000 +wifi-densepose start + +# Serve the UI on port 3000 +cd ui/ python -m http.server 3000 -# Open http://localhost:3000/tests/test-runner.html ``` +Open http://localhost:3000 -### Test Categories -- **API Configuration Tests** - Configuration and URL building -- **API Service Tests** - HTTP client functionality -- **WebSocket Service Tests** - WebSocket connection management -- **Pose Service Tests** - Pose estimation API wrapper -- **Health Service Tests** - Health monitoring functionality -- **UI Component Tests** - Component behavior and interaction -- **Integration Tests** - End-to-end functionality +## Pose Estimation Modes -### Integration Testing -Use `tests/integration-test.html` for visual integration testing: +| Mode | Badge | Requirements | Accuracy | +|------|-------|-------------|----------| +| **Signal-Derived** | Green | 1+ ESP32, no model needed | Presence, breathing, gross motion | +| **Model Inference** | Blue | 4+ ESP32s + trained `.rvf` model | Full 17-keypoint COCO pose | +To use model inference, start the server with a trained model: ```bash -# Open http://localhost:3000/tests/integration-test.html +sensing-server --source esp32 --model path/to/model.rvf --ui-path ./ui ``` -Features: -- Mock server with realistic API responses -- Visual testing of all components -- Real-time data simulation -- Error scenario testing -- WebSocket stream testing - -## πŸ› οΈ Usage - -### Basic Setup -```html - - - - - - -
- -
- - - -``` - -### Using Services -```javascript -import { poseService } from './services/pose.service.js'; -import { healthService } from './services/health.service.js'; - -// Get current pose data -const poseData = await poseService.getCurrentPose(); - -// Subscribe to health updates -healthService.subscribeToHealth(health => { - console.log('Health status:', health.status); -}); - -// Start pose streaming -poseService.startPoseStream({ - minConfidence: 0.7, - maxFps: 30 -}); - -poseService.subscribeToPoseUpdates(update => { - if (update.type === 'pose_update') { - console.log('New pose data:', update.data); - } -}); -``` - -### Using Components -```javascript -import { TabManager } from './components/TabManager.js'; -import { DashboardTab } from './components/DashboardTab.js'; - -// Initialize tab manager -const container = document.querySelector('.container'); -const tabManager = new TabManager(container); -tabManager.init(); - -// Initialize dashboard -const dashboardContainer = document.getElementById('dashboard'); -const dashboard = new DashboardTab(dashboardContainer); -await dashboard.init(); -``` - -## πŸ”§ Configuration +## Configuration ### API Configuration -Edit `config/api.config.js` to modify API settings: +Edit `config/api.config.js`: ```javascript export const API_CONFIG = { BASE_URL: window.location.origin, API_VERSION: '/api/v1', - - // Rate limiting - RATE_LIMITS: { - REQUESTS_PER_MINUTE: 60, - BURST_LIMIT: 10 - }, - - // WebSocket configuration WS_CONFIG: { RECONNECT_DELAY: 5000, - MAX_RECONNECT_ATTEMPTS: 5, + MAX_RECONNECT_ATTEMPTS: 20, PING_INTERVAL: 30000 } }; ``` -### Authentication -```javascript -import { apiService } from './services/api.service.js'; +## Testing -// Set authentication token -apiService.setAuthToken('your-jwt-token'); +Open `tests/test-runner.html` to run the test suite: -// Add request interceptor for auth -apiService.addRequestInterceptor((url, options) => { - // Modify request before sending - return { url, options }; -}); -``` - -## 🎨 Styling - -The UI uses a comprehensive CSS design system with: - -- CSS Custom Properties for theming -- Dark/light mode support -- Responsive design -- Component-based styling -- Smooth animations and transitions - -### Key CSS Variables -```css -:root { - --color-primary: rgba(33, 128, 141, 1); - --color-background: rgba(252, 252, 249, 1); - --color-surface: rgba(255, 255, 253, 1); - --color-text: rgba(19, 52, 59, 1); - --space-16: 16px; - --radius-lg: 12px; -} -``` - -## πŸ” Monitoring & Debugging - -### Health Monitoring -```javascript -import { healthService } from './services/health.service.js'; - -// Start automatic health checks -healthService.startHealthMonitoring(30000); // Every 30 seconds - -// Check if system is healthy -const isHealthy = healthService.isSystemHealthy(); - -// Get specific component status -const apiStatus = healthService.getComponentStatus('api'); -``` - -### Error Handling -```javascript -// Global error handling -window.addEventListener('error', (event) => { - console.error('Global error:', event.error); -}); - -// API error handling -apiService.addResponseInterceptor(async (response, url) => { - if (!response.ok) { - console.error(`API error: ${response.status} for ${url}`); - } - return response; -}); -``` - -## πŸš€ Deployment - -### Development - -**Option 1: Use the startup script** ```bash -cd /workspaces/wifi-densepose/ui -./start-ui.sh -``` - -**Option 2: Manual setup** -```bash -# First, start your FastAPI backend (runs on port 8000) -wifi-densepose start -# or from the main project directory: -python -m wifi_densepose.main - -# Then, start the UI server on a different port to avoid conflicts -cd /workspaces/wifi-densepose/ui +cd ui/ python -m http.server 3000 -# or -npx http-server . -p 3000 - -# Open the UI at http://localhost:3000 -# The UI will automatically detect and connect to your backend +# Open http://localhost:3000/tests/test-runner.html ``` -### Backend Detection Behavior -- **Real Backend Available**: UI connects to `http://localhost:8000` and shows βœ… "Connected to real backend" -- **Backend Unavailable**: UI automatically uses mock server and shows ⚠️ "Mock server active - testing mode" -- **Force Mock Mode**: Set `API_CONFIG.MOCK_SERVER.ENABLED = true` for testing +Test categories: API configuration, API service, WebSocket, pose service, health service, UI components, integration. -### Production -1. Configure `API_CONFIG.BASE_URL` for your backend -2. Set up HTTPS for WebSocket connections -3. Configure authentication if required -4. Optimize assets (minify CSS/JS) -5. Set up monitoring and logging +## Styling -## 🀝 Contributing +Uses a CSS design system with custom properties, dark/light mode, responsive layout, and component-based styling. Key variables in `:root` of `style.css`. -1. Follow the modular architecture -2. Add tests for new functionality -3. Update documentation -4. Ensure TypeScript compatibility -5. Test with mock server +## License -## πŸ“„ License - -This project is part of the WiFi-DensePose system. See the main project LICENSE file for details. \ No newline at end of file +Part of the WiFi-DensePose system. See the main project LICENSE file. diff --git a/ui/components/LiveDemoTab.js b/ui/components/LiveDemoTab.js index 9997d54e..bf19111f 100644 --- a/ui/components/LiveDemoTab.js +++ b/ui/components/LiveDemoTab.js @@ -14,7 +14,9 @@ export class LiveDemoTab { currentZone: 'zone_1', debugMode: false, autoReconnect: true, - renderMode: 'skeleton' + renderMode: 'skeleton', + // 'unknown' | 'signal_derived' | 'model_inference' + poseSource: 'unknown' }; this.components = { @@ -136,6 +138,48 @@ export class LiveDemoTab { +
+

Estimation Mode

+
+ Unknown +

+ Waiting for first frame... +

+
+
+ +
+

Setup Guide

+
+
+ 1x +
+ 1 ESP32 + 1 AP +

Presence, breathing, gross motion

+
+
+
+ 3x +
+ 2-3 ESP32s +

Body localization, motion direction

+
+
+
+ 4x+ +
+ 4+ ESP32s + trained model +

Individual limb tracking, full pose

+
+
+
+

+ Signal-Derived mode uses aggregate CSI features. + For per-limb tracking, load a trained .rvf model + with --model path.rvf and use 4+ sensors. +

+
+

System Health

@@ -185,8 +229,8 @@ export class LiveDemoTab { flex-direction: column; height: 100%; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Oxygen, Ubuntu, Cantarell, sans-serif; - background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); - color: #333; + background: #0a0f1a; + color: #e0e0e0; } .demo-header { @@ -194,10 +238,10 @@ export class LiveDemoTab { justify-content: space-between; align-items: center; padding: 20px 24px; - background: rgba(255, 255, 255, 0.95); + background: rgba(15, 20, 35, 0.95); backdrop-filter: blur(10px); - border-bottom: 1px solid rgba(255, 255, 255, 0.2); - box-shadow: 0 2px 20px rgba(0, 0, 0, 0.1); + border-bottom: 1px solid rgba(255, 255, 255, 0.08); + box-shadow: 0 2px 20px rgba(0, 0, 0, 0.3); position: relative; z-index: 10; } @@ -210,10 +254,10 @@ export class LiveDemoTab { .demo-title h2 { margin: 0; - color: #333; + color: #e0e0e0; font-size: 22px; font-weight: 700; - background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + background: linear-gradient(135deg, #667eea 0%, #a78bfa 100%); -webkit-background-clip: text; -webkit-text-fill-color: transparent; background-clip: text; @@ -224,9 +268,9 @@ export class LiveDemoTab { align-items: center; gap: 10px; padding: 8px 16px; - background: rgba(248, 249, 250, 0.8); + background: rgba(30, 40, 60, 0.8); border-radius: 20px; - border: 1px solid rgba(222, 226, 230, 0.5); + border: 1px solid rgba(255, 255, 255, 0.1); } .status-indicator { @@ -260,7 +304,7 @@ export class LiveDemoTab { .status-text { font-size: 13px; font-weight: 500; - color: #495057; + color: #b0b8c8; } .demo-controls { @@ -294,19 +338,19 @@ export class LiveDemoTab { .btn--primary:hover:not(:disabled) { transform: translateY(-2px); - box-shadow: 0 4px 16px rgba(102, 126, 234, 0.3); + box-shadow: 0 4px 16px rgba(102, 126, 234, 0.4); } .btn--secondary { - background: #f8f9fa; - color: #495057; - border-color: #dee2e6; + background: rgba(30, 40, 60, 0.8); + color: #b0b8c8; + border-color: rgba(255, 255, 255, 0.1); } .btn--secondary:hover:not(:disabled) { - background: #e9ecef; + background: rgba(40, 50, 75, 0.9); transform: translateY(-1px); - box-shadow: 0 4px 12px rgba(0, 0, 0, 0.15); + box-shadow: 0 4px 12px rgba(0, 0, 0, 0.3); } .btn:disabled { @@ -324,19 +368,20 @@ export class LiveDemoTab { .zone-select { padding: 10px 14px; - border: 1px solid #dee2e6; + border: 1px solid rgba(255, 255, 255, 0.1); border-radius: 8px; - background: white; + background: rgba(30, 40, 60, 0.8); + color: #b0b8c8; font-size: 14px; cursor: pointer; - box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1); + box-shadow: 0 2px 8px rgba(0, 0, 0, 0.2); transition: all 0.2s ease; } .zone-select:focus { outline: none; border-color: #667eea; - box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1); + box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.2); } .demo-content { @@ -344,18 +389,17 @@ export class LiveDemoTab { flex: 1; gap: 24px; padding: 24px; - background: rgba(255, 255, 255, 0.1); - backdrop-filter: blur(10px); + background: #0a0f1a; } .demo-main { flex: 2; min-height: 500px; - background: white; + background: #111827; border-radius: 12px; overflow: hidden; - box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1); - border: 1px solid rgba(255, 255, 255, 0.2); + box-shadow: 0 8px 32px rgba(0, 0, 0, 0.3); + border: 1px solid rgba(255, 255, 255, 0.06); } .pose-detection-container { @@ -372,15 +416,15 @@ export class LiveDemoTab { } .metrics-panel, .health-panel, .debug-panel { - background: #fff; - border: 1px solid #ddd; + background: rgba(17, 24, 39, 0.9); + border: 1px solid rgba(255, 255, 255, 0.08); border-radius: 8px; padding: 15px; } .metrics-panel h4, .health-panel h4, .debug-panel h4 { margin: 0 0 15px 0; - color: #333; + color: #e0e0e0; font-size: 14px; font-weight: 600; } @@ -394,12 +438,12 @@ export class LiveDemoTab { } .metric label, .health-check label { - color: #666; + color: #8899aa; } .metric span, .health-check span { font-weight: 500; - color: #333; + color: #c8d0dc; } .debug-actions { @@ -411,18 +455,20 @@ export class LiveDemoTab { .debug-info textarea { width: 100%; - border: 1px solid #ddd; + border: 1px solid rgba(255, 255, 255, 0.1); border-radius: 4px; padding: 8px; font-family: monospace; font-size: 11px; resize: vertical; + background: #0a0f1a; + color: #c8d0dc; } .error-display { - background: #f8d7da; - color: #721c24; - border: 1px solid #f5c6cb; + background: rgba(220, 53, 69, 0.15); + color: #f5a0a8; + border: 1px solid rgba(220, 53, 69, 0.3); border-radius: 4px; padding: 12px; margin: 10px 20px; @@ -432,6 +478,134 @@ export class LiveDemoTab { .health-good { color: #28a745; } .health-poor { color: #ffc107; } .health-bad { color: #dc3545; } + + /* Pose estimation mode indicator */ + .pose-source-panel { + background: rgba(17, 24, 39, 0.9); + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 8px; + padding: 15px; + } + + .pose-source-panel h4 { + margin: 0 0 12px 0; + color: #e0e0e0; + font-size: 14px; + font-weight: 600; + } + + .pose-source-indicator { + display: flex; + flex-direction: column; + gap: 8px; + } + + .pose-source-badge { + display: inline-block; + padding: 4px 12px; + border-radius: 12px; + font-size: 12px; + font-weight: 600; + text-transform: uppercase; + letter-spacing: 0.5px; + width: fit-content; + } + + .pose-source-unknown { + background: rgba(108, 117, 125, 0.15); + color: #8899aa; + border: 1px solid rgba(108, 117, 125, 0.3); + } + + .pose-source-signal { + background: rgba(0, 204, 136, 0.12); + color: #00cc88; + border: 1px solid rgba(0, 204, 136, 0.3); + } + + .pose-source-model { + background: rgba(102, 126, 234, 0.12); + color: #8ea4f0; + border: 1px solid rgba(102, 126, 234, 0.3); + } + + .pose-source-description { + margin: 0; + font-size: 11px; + color: #8899aa; + line-height: 1.4; + } + + .setup-guide-panel { + background: rgba(17, 24, 39, 0.9); + border: 1px solid rgba(255, 255, 255, 0.08); + border-radius: 8px; + padding: 15px; + } + + .setup-guide-panel h4 { + margin: 0 0 12px 0; + color: #e0e0e0; + font-size: 14px; + font-weight: 600; + } + + .setup-levels { + display: flex; + flex-direction: column; + gap: 10px; + } + + .setup-level { + display: flex; + align-items: center; + gap: 10px; + padding: 8px; + border-radius: 6px; + background: rgba(30, 40, 60, 0.6); + border: 1px solid rgba(255, 255, 255, 0.06); + } + + .setup-level-icon { + background: linear-gradient(135deg, #667eea 0%, #764ba2 100%); + color: white; + font-size: 11px; + font-weight: 700; + width: 32px; + height: 32px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + } + + .setup-level-info strong { + font-size: 12px; + color: #c8d0dc; + display: block; + } + + .setup-level-info p { + margin: 2px 0 0; + font-size: 11px; + color: #8899aa; + } + + .setup-note { + margin: 10px 0 0; + font-size: 11px; + color: #6b7a8d; + line-height: 1.5; + } + + .setup-note code { + background: rgba(102, 126, 234, 0.12); + color: #8ea4f0; + padding: 1px 4px; + border-radius: 3px; + font-size: 10px; + } `; if (!document.querySelector('#live-demo-enhanced-styles')) { @@ -545,7 +719,11 @@ export class LiveDemoTab { handlePoseUpdate(data) { this.metrics.frameCount++; this.metrics.lastUpdate = Date.now(); - this.updateDebugOutput(`Pose update: ${data.persons?.length || 0} persons detected`); + // Update pose source indicator if the backend supplies it + if (data.pose_source && data.pose_source !== this.state.poseSource) { + this.setState({ poseSource: data.pose_source }); + } + this.updateDebugOutput(`Pose update: ${data.persons?.length || 0} persons detected (${data.pose_source || 'unknown'})`); } handleCanvasError(error) { @@ -706,6 +884,7 @@ export class LiveDemoTab { this.updateStatusIndicator(); this.updateControls(); this.updateMetricsDisplay(); + this.updatePoseSourceIndicator(); } updateStatusIndicator() { @@ -789,6 +968,33 @@ export class LiveDemoTab { } } + updatePoseSourceIndicator() { + const badge = this.container.querySelector('#pose-source-badge'); + const description = this.container.querySelector('#pose-source-description'); + + if (!badge || !description) return; + + const source = this.state.poseSource; + + if (source === 'model_inference') { + badge.className = 'pose-source-badge pose-source-model'; + badge.textContent = 'Model Inference'; + description.textContent = + 'Pose is estimated by a trained neural network ' + + 'loaded from an RVF container.'; + } else if (source === 'signal_derived') { + badge.className = 'pose-source-badge pose-source-signal'; + badge.textContent = 'Signal-Derived'; + description.textContent = + 'Keypoints are derived from live CSI signal features ' + + '(motion power, breathing rate, variance).'; + } else { + badge.className = 'pose-source-badge pose-source-unknown'; + badge.textContent = 'Unknown'; + description.textContent = 'Waiting for first frame...'; + } + } + getHealthClass(status) { switch (status) { case 'connected': return 'good'; diff --git a/ui/components/PoseDetectionCanvas.js b/ui/components/PoseDetectionCanvas.js index 62f98149..cc267eba 100644 --- a/ui/components/PoseDetectionCanvas.js +++ b/ui/components/PoseDetectionCanvas.js @@ -89,21 +89,17 @@ export class PoseDetectionCanvas {
-
- - - - -
-
- - -
+ + + + + +
@@ -124,20 +120,20 @@ export class PoseDetectionCanvas { const style = document.createElement('style'); style.textContent = ` .pose-detection-canvas-wrapper { - border: 1px solid #ddd; + border: 1px solid rgba(255, 255, 255, 0.06); border-radius: 8px; overflow: hidden; - background: #f9f9f9; - font-family: Arial, sans-serif; + background: #0d1117; + font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif; } .pose-canvas-header { display: flex; justify-content: space-between; align-items: center; - padding: 10px 15px; - background: #f0f0f0; - border-bottom: 1px solid #ddd; + padding: 12px 16px; + background: rgba(15, 20, 35, 0.95); + border-bottom: 1px solid rgba(255, 255, 255, 0.06); } .pose-canvas-title { @@ -148,156 +144,166 @@ export class PoseDetectionCanvas { .pose-canvas-title h3 { margin: 0; - color: #333; + color: #e0e0e0; font-size: 16px; + font-weight: 600; } .connection-status { display: flex; align-items: center; - gap: 5px; + gap: 6px; + padding: 4px 10px; + background: rgba(30, 40, 60, 0.6); + border-radius: 12px; + border: 1px solid rgba(255, 255, 255, 0.06); } .status-indicator { - width: 10px; - height: 10px; + width: 8px; + height: 8px; border-radius: 50%; - background: #ccc; + background: #4a5568; transition: background-color 0.3s; } - .status-indicator.connected { background: #28a745; } - .status-indicator.connecting { background: #ffc107; } - .status-indicator.error { background: #dc3545; } - .status-indicator.disconnected { background: #6c757d; } + .status-indicator.connected { background: #00cc88; box-shadow: 0 0 6px rgba(0, 204, 136, 0.5); } + .status-indicator.connecting { background: #fbbf24; box-shadow: 0 0 6px rgba(251, 191, 36, 0.5); animation: pulse 1.5s ease-in-out infinite; } + .status-indicator.error { background: #ef4444; box-shadow: 0 0 6px rgba(239, 68, 68, 0.5); } + .status-indicator.disconnected { background: #4a5568; } .status-text { - font-size: 12px; - color: #666; - min-width: 80px; + font-size: 11px; + color: #8899aa; + min-width: 70px; + text-transform: uppercase; + letter-spacing: 0.5px; + font-weight: 500; } .pose-canvas-controls { - display: flex; - align-items: center; - justify-content: space-between; - gap: 15px; - flex-wrap: wrap; - } - - .control-group { display: flex; align-items: center; gap: 8px; - } - - .primary-controls { - flex: 1; - } - - .secondary-controls { - flex-shrink: 0; + flex-wrap: nowrap; } .btn { padding: 8px 16px; - border: 1px solid #ddd; - border-radius: 6px; - background: #ffffff; - color: #333333; + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 8px; + background: rgba(30, 40, 60, 0.8); + color: #c8d0dc; cursor: pointer; font-size: 13px; font-weight: 500; transition: all 0.2s ease; - box-shadow: 0 1px 3px rgba(0,0,0,0.1); + box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2); text-decoration: none; - display: inline-block; + display: inline-flex; + align-items: center; + gap: 4px; min-width: 80px; - text-align: center; + justify-content: center; } .btn:hover:not(:disabled) { - background: #f8f9fa; - border-color: #adb5bd; - box-shadow: 0 2px 6px rgba(0,0,0,0.15); transform: translateY(-1px); + box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3); } .btn:active:not(:disabled) { transform: translateY(0); - box-shadow: 0 1px 3px rgba(0,0,0,0.1); + box-shadow: 0 1px 3px rgba(0, 0, 0, 0.2); } .btn:disabled { - opacity: 0.6; + opacity: 0.35; cursor: not-allowed; - background: #e9ecef; - color: #6c757d; + background: rgba(20, 30, 50, 0.6); + color: #4a5568; transform: none !important; box-shadow: none !important; } - .btn-start { - background: #28a745; - color: white; - border-color: #28a745; + .btn-start { + background: rgba(0, 204, 136, 0.15); + color: #00cc88; + border-color: rgba(0, 204, 136, 0.3); } - .btn-start:hover:not(:disabled) { - background: #218838; - border-color: #1e7e34; + .btn-start:hover:not(:disabled) { + background: rgba(0, 204, 136, 0.25); + border-color: rgba(0, 204, 136, 0.5); + box-shadow: 0 4px 12px rgba(0, 204, 136, 0.2); } - .btn-stop { - background: #dc3545; - color: white; - border-color: #dc3545; + .btn-stop { + background: rgba(239, 68, 68, 0.15); + color: #ef4444; + border-color: rgba(239, 68, 68, 0.3); } - .btn-stop:hover:not(:disabled) { - background: #c82333; - border-color: #bd2130; + .btn-stop:hover:not(:disabled) { + background: rgba(239, 68, 68, 0.25); + border-color: rgba(239, 68, 68, 0.5); + box-shadow: 0 4px 12px rgba(239, 68, 68, 0.2); } - .btn-reconnect { - background: #17a2b8; - color: white; - border-color: #17a2b8; + .btn-reconnect { + background: rgba(59, 130, 246, 0.15); + color: #60a5fa; + border-color: rgba(59, 130, 246, 0.3); } - .btn-reconnect:hover:not(:disabled) { - background: #138496; - border-color: #117a8b; + .btn-reconnect:hover:not(:disabled) { + background: rgba(59, 130, 246, 0.25); + border-color: rgba(59, 130, 246, 0.5); + box-shadow: 0 4px 12px rgba(59, 130, 246, 0.2); } - .btn-demo { - background: #6f42c1; - color: white; - border-color: #6f42c1; + .btn-demo { + background: rgba(139, 92, 246, 0.15); + color: #a78bfa; + border-color: rgba(139, 92, 246, 0.3); } - .btn-demo:hover:not(:disabled) { - background: #5a32a3; - border-color: #512a97; + .btn-demo:hover:not(:disabled) { + background: rgba(139, 92, 246, 0.25); + border-color: rgba(139, 92, 246, 0.5); + box-shadow: 0 4px 12px rgba(139, 92, 246, 0.2); } - .btn-settings { - background: #6c757d; - color: white; - border-color: #6c757d; + .btn-settings { + background: rgba(100, 116, 139, 0.15); + color: #94a3b8; + border-color: rgba(100, 116, 139, 0.3); } - .btn-settings:hover:not(:disabled) { - background: #5a6268; - border-color: #545b62; + .btn-settings:hover:not(:disabled) { + background: rgba(100, 116, 139, 0.25); + border-color: rgba(100, 116, 139, 0.5); } .mode-select { - padding: 5px 8px; - border: 1px solid #ddd; - border-radius: 4px; - background: #fff; - font-size: 12px; + padding: 8px 12px; + border: 1px solid rgba(255, 255, 255, 0.1); + border-radius: 8px; + background: rgba(30, 40, 60, 0.8); + color: #b0b8c8; + font-size: 13px; + cursor: pointer; + } + + .mode-select:focus { + outline: none; + border-color: rgba(139, 92, 246, 0.5); + box-shadow: 0 0 0 2px rgba(139, 92, 246, 0.15); + } + + .mode-select option { + background: #1a2234; + color: #c8d0dc; } .pose-canvas-container { diff --git a/ui/components/SensingTab.js b/ui/components/SensingTab.js index ba4d167d..c2895e02 100644 --- a/ui/components/SensingTab.js +++ b/ui/components/SensingTab.js @@ -33,6 +33,13 @@ export class SensingTab { _buildDOM() { this.container.innerHTML = `

Live WiFi Sensing

+ + +
+ RECONNECTING... +
+
@@ -98,6 +105,17 @@ export class SensingTab {
+ +
+
About This Data
+

+ Metrics are computed from WiFi Channel State Information (CSI). + With 1 ESP32 you get presence detection, breathing + estimation, and gross motion. Add 3-4+ ESP32 nodes + around the room for spatial resolution and limb-level tracking. +

+
+
Details
@@ -178,19 +196,34 @@ export class SensingTab { } _onStateChange(state) { - const dot = this.container.querySelector('#sensingDot'); - const text = this.container.querySelector('#sensingState'); - if (!dot || !text) return; + const dot = this.container.querySelector('#sensingDot'); + const text = this.container.querySelector('#sensingState'); + const banner = this.container.querySelector('#sensingSourceBanner'); - const labels = { - disconnected: 'Disconnected', - connecting: 'Connecting...', - connected: 'Connected', - simulated: 'Simulated', - }; + if (dot && text) { + const stateLabels = { + disconnected: 'Disconnected', + connecting: 'Connecting...', + connected: 'Connected', + reconnecting: 'Reconnecting...', + simulated: 'Simulated', + }; + dot.className = 'sensing-dot ' + state; + text.textContent = stateLabels[state] || state; + } - dot.className = 'sensing-dot ' + state; - text.textContent = labels[state] || state; + if (banner) { + // Map the service's dataSource to banner text and CSS modifier class. + const dataSource = sensingService.dataSource; + const bannerConfig = { + live: { text: 'LIVE - ESP32', cls: 'sensing-source-live' }, + reconnecting: { text: 'RECONNECTING...', cls: 'sensing-source-reconnecting' }, + simulated: { text: 'SIMULATED DATA', cls: 'sensing-source-simulated' }, + }; + const cfg = bannerConfig[dataSource] || bannerConfig.reconnecting; + banner.textContent = cfg.text; + banner.className = 'sensing-source-banner ' + cfg.cls; + } } // ---- HUD update -------------------------------------------------------- diff --git a/ui/mobile/src/hooks/usePoseStream.ts b/ui/mobile/src/hooks/usePoseStream.ts index bebab716..c3fe8046 100644 --- a/ui/mobile/src/hooks/usePoseStream.ts +++ b/ui/mobile/src/hooks/usePoseStream.ts @@ -1,6 +1,7 @@ import { useEffect } from 'react'; import { wsService } from '@/services/ws.service'; import { usePoseStore } from '@/stores/poseStore'; +import { useSettingsStore } from '@/stores/settingsStore'; export interface UsePoseStreamResult { connectionStatus: ReturnType['connectionStatus']; @@ -12,16 +13,20 @@ export function usePoseStream(): UsePoseStreamResult { const connectionStatus = usePoseStore((state) => state.connectionStatus); const lastFrame = usePoseStore((state) => state.lastFrame); const isSimulated = usePoseStore((state) => state.isSimulated); + const serverUrl = useSettingsStore((state) => state.serverUrl); useEffect(() => { const unsubscribe = wsService.subscribe((frame) => { usePoseStore.getState().handleFrame(frame); }); + // Auto-connect to sensing server on mount + wsService.connect(serverUrl); + return () => { unsubscribe(); }; - }, []); + }, [serverUrl]); return { connectionStatus, lastFrame, isSimulated }; } diff --git a/ui/mobile/src/screens/LiveScreen/GaussianSplatWebView.web.tsx b/ui/mobile/src/screens/LiveScreen/GaussianSplatWebView.web.tsx index 3a9ca43f..850db965 100644 --- a/ui/mobile/src/screens/LiveScreen/GaussianSplatWebView.web.tsx +++ b/ui/mobile/src/screens/LiveScreen/GaussianSplatWebView.web.tsx @@ -3,40 +3,116 @@ import { StyleSheet, View } from 'react-native'; import * as THREE from 'three'; import type { SensingFrame } from '@/types/sensing'; -type GaussianSplatWebViewWebProps = { +type Props = { onReady: () => void; onFps: (fps: number) => void; onError: (msg: string) => void; frame: SensingFrame | null; }; +// COCO skeleton bones const BONES: [number, number][] = [ [0,1],[0,2],[1,3],[2,4],[5,6],[5,7],[7,9],[6,8],[8,10], [5,11],[6,12],[11,12],[11,13],[13,15],[12,14],[14,16], ]; -export const GaussianSplatWebViewWeb = ({ onReady, onFps, onError, frame }: GaussianSplatWebViewWebProps) => { - const containerRef = useRef(null); - const sceneRef = useRef<{ - renderer: THREE.WebGLRenderer; - scene: THREE.Scene; - camera: THREE.PerspectiveCamera; - joints: THREE.Mesh[]; - boneLines: { line: THREE.Line; a: number; b: number }[]; - ring: THREE.Mesh; - particleGeo: THREE.BufferGeometry; - pointLight: THREE.PointLight; - animId: number; - cameraAngle: number; - cameraRadius: number; - cameraY: number; - isDragging: boolean; - frameCount: number; - lastFpsTime: number; - } | null>(null); - const frameRef = useRef(null); +// Standing pose (meters, Y-up) +const BASE_POSE: [number, number, number][] = [ + [ 0.00, 1.72, 0.04], // 0 nose + [-0.03, 1.76, 0.05], // 1 left eye + [ 0.03, 1.76, 0.05], // 2 right eye + [-0.08, 1.74,-0.01], // 3 left ear + [ 0.08, 1.74,-0.01], // 4 right ear + [-0.20, 1.45, 0.00], // 5 left shoulder + [ 0.20, 1.45, 0.00], // 6 right shoulder + [-0.26, 1.12, 0.04], // 7 left elbow + [ 0.26, 1.12, 0.04], // 8 right elbow + [-0.28, 0.82, 0.02], // 9 left wrist + [ 0.28, 0.82, 0.02], // 10 right wrist + [-0.11, 0.95, 0.00], // 11 left hip + [ 0.11, 0.95, 0.00], // 12 right hip + [-0.12, 0.50, 0.02], // 13 left knee + [ 0.12, 0.50, 0.02], // 14 right knee + [-0.12, 0.04, 0.00], // 15 left ankle + [ 0.12, 0.04, 0.00], // 16 right ankle +]; - // Keep frame ref current without re-running effect +// DensePose-style body part colors (24 parts β†’ simplified per-segment) +const DENSEPOSE_COLORS: Record = { + head: 0xf4a582, // warm skin + neck: 0xd6604d, // darker warm + torsoFront: 0x92c5de, // blue-gray + torsoSide: 0x4393c3, // steel blue + pelvis: 0x2166ac, // deep blue + lUpperArm: 0xd73027, // red + rUpperArm: 0xf46d43, // orange-red + lForearm: 0xfdae61, // orange + rForearm: 0xfee090, // light orange + lHand: 0xffffbf, // pale yellow + rHand: 0xffffbf, + lThigh: 0xa6d96a, // green + rThigh: 0x66bd63, // darker green + lShin: 0x1a9850, // deep green + rShin: 0x006837, // forest + lFoot: 0x762a83, // purple + rFoot: 0x9970ab, // light purple +}; + +// Body segments: [jointA, jointB, topRadius, botRadius, colorKey] +const BODY_SEGS: [number, number, number, number, string][] = [ + [5, 6, 0.10, 0.10, 'torsoFront'], // collar + [5, 11, 0.09, 0.07, 'torsoSide'], // L torso + [6, 12, 0.09, 0.07, 'torsoSide'], // R torso + [11, 12, 0.08, 0.08, 'pelvis'], // pelvis + [5, 7, 0.045,0.040,'lUpperArm'], // L upper arm + [7, 9, 0.038,0.032,'lForearm'], // L forearm + [6, 8, 0.045,0.040,'rUpperArm'], // R upper arm + [8, 10, 0.038,0.032,'rForearm'], // R forearm + [11, 13, 0.065,0.050,'lThigh'], // L thigh + [13, 15, 0.048,0.038,'lShin'], // L shin + [12, 14, 0.065,0.050,'rThigh'], // R thigh + [14, 16, 0.048,0.038,'rShin'], // R shin +]; + +function makePart(scene: THREE.Scene, rTop: number, rBot: number, color: number, glow: boolean = false): THREE.Mesh { + const geo = new THREE.CapsuleGeometry((rTop + rBot) / 2, 1, 6, 12); + const mat = new THREE.MeshPhysicalMaterial({ + color, emissive: color, + emissiveIntensity: glow ? 0.4 : 0.08, + transparent: true, opacity: glow ? 0.12 : 0.85, + roughness: 0.35, metalness: 0.1, + clearcoat: glow ? 0 : 0.3, clearcoatRoughness: 0.4, + side: glow ? THREE.BackSide : THREE.FrontSide, + }); + const m = new THREE.Mesh(geo, mat); + m.visible = false; + m.castShadow = !glow; + scene.add(m); + return m; +} + +function positionLimb(mesh: THREE.Mesh, a: THREE.Vector3, b: THREE.Vector3, rTop: number, rBot: number) { + const mid = new THREE.Vector3().addVectors(a, b).multiplyScalar(0.5); + mesh.position.copy(mid); + const len = a.distanceTo(b); + // CapsuleGeometry height param = 1, so scale Y to actual length + mesh.scale.set((rTop + rBot) * 10, len, (rTop + rBot) * 10); + const dir = new THREE.Vector3().subVectors(b, a).normalize(); + const up = new THREE.Vector3(0, 1, 0); + const quat = new THREE.Quaternion().setFromUnitVectors(up, dir); + mesh.quaternion.copy(quat); +} + +function lerp3(out: THREE.Vector3, target: THREE.Vector3, alpha: number) { + out.x += (target.x - out.x) * alpha; + out.y += (target.y - out.y) * alpha; + out.z += (target.z - out.z) * alpha; +} + +export const GaussianSplatWebViewWeb = ({ onReady, onFps, onError, frame }: Props) => { + const containerRef = useRef(null); + const frameRef = useRef(null); + const sceneRef = useRef(null); frameRef.current = frame; const cleanup = useCallback(() => { @@ -44,11 +120,11 @@ export const GaussianSplatWebViewWeb = ({ onReady, onFps, onError, frame }: Gaus if (!s) return; cancelAnimationFrame(s.animId); s.renderer.dispose(); - s.scene.traverse((obj) => { - if (obj instanceof THREE.Mesh) { - obj.geometry.dispose(); - if (Array.isArray(obj.material)) obj.material.forEach((m) => m.dispose()); - else obj.material.dispose(); + s.scene.traverse((obj: any) => { + if (obj.geometry) obj.geometry.dispose(); + if (obj.material) { + const mats = Array.isArray(obj.material) ? obj.material : [obj.material]; + mats.forEach((m: any) => m.dispose()); } }); sceneRef.current = null; @@ -57,222 +133,544 @@ export const GaussianSplatWebViewWeb = ({ onReady, onFps, onError, frame }: Gaus useEffect(() => { const container = containerRef.current; if (!container) return; - try { const W = () => container.clientWidth || window.innerWidth; const H = () => container.clientHeight || window.innerHeight; - // Renderer - const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: false }); + // --- Renderer --- + const renderer = new THREE.WebGLRenderer({ antialias: true, powerPreference: 'high-performance' }); renderer.setSize(W(), H()); renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2)); - renderer.setClearColor(0x0a0e1a); + renderer.setClearColor(0x080c16); + renderer.shadowMap.enabled = true; + renderer.shadowMap.type = THREE.PCFSoftShadowMap; + renderer.toneMapping = THREE.ACESFilmicToneMapping; + renderer.toneMappingExposure = 1.1; container.appendChild(renderer.domElement); - // Scene const scene = new THREE.Scene(); - scene.background = new THREE.Color(0x0a0e1a); - scene.fog = new THREE.FogExp2(0x0a0e1a, 0.008); + scene.background = new THREE.Color(0x080c16); + scene.fog = new THREE.FogExp2(0x080c16, 0.018); - // Camera - const camera = new THREE.PerspectiveCamera(60, W() / H(), 0.1, 500); - camera.position.set(0, 2, 6); - camera.lookAt(0, 1, 0); + const camera = new THREE.PerspectiveCamera(45, W() / H(), 0.1, 200); + camera.position.set(0, 1.4, 3.5); + camera.lookAt(0, 0.9, 0); - // Grid - const grid = new THREE.GridHelper(20, 40, 0x1a3a4a, 0x0d1f2a); - scene.add(grid); + // --- Lighting (3-point + rim) --- + scene.add(new THREE.AmbientLight(0x223344, 0.5)); - // Lights - scene.add(new THREE.AmbientLight(0x32b8c6, 0.3)); - const pointLight = new THREE.PointLight(0x32b8c6, 1.5, 20); - pointLight.position.set(0, 4, 0); - scene.add(pointLight); + const key = new THREE.DirectionalLight(0xddeeff, 1.0); + key.position.set(2, 5, 3); + key.castShadow = true; + key.shadow.mapSize.set(1024, 1024); + key.shadow.camera.near = 0.5; + key.shadow.camera.far = 15; + key.shadow.camera.left = -3; + key.shadow.camera.right = 3; + key.shadow.camera.top = 3; + key.shadow.camera.bottom = -1; + scene.add(key); - // Skeleton joints (17 COCO keypoints) - const jointGeo = new THREE.SphereGeometry(0.06, 8, 8); - const joints: THREE.Mesh[] = []; - for (let i = 0; i < 17; i++) { - const mat = new THREE.MeshStandardMaterial({ - color: 0x32b8c6, - emissive: 0x32b8c6, - emissiveIntensity: 0.6, - }); - const m = new THREE.Mesh(jointGeo, mat); - m.visible = false; - scene.add(m); - joints.push(m); + const rim = new THREE.PointLight(0x32b8c6, 1.5, 12); + rim.position.set(-1.5, 2.5, -2); + scene.add(rim); + + const fill = new THREE.PointLight(0x554488, 0.5, 8); + fill.position.set(1.5, 0.8, 2.5); + scene.add(fill); + + const under = new THREE.PointLight(0x225566, 0.4, 5); + under.position.set(0, 0.1, 1); + scene.add(under); + + // --- Ground --- + const groundGeo = new THREE.PlaneGeometry(20, 20); + const groundMat = new THREE.MeshStandardMaterial({ + color: 0x0a0e1a, roughness: 0.9, metalness: 0.1, + }); + const ground = new THREE.Mesh(groundGeo, groundMat); + ground.rotation.x = -Math.PI / 2; + ground.receiveShadow = true; + scene.add(ground); + + const gridH = new THREE.GridHelper(20, 40, 0x1a3050, 0x0e1826); + gridH.position.y = 0.002; + scene.add(gridH); + + // --- Signal field (20x20) --- + const GS = 20; + const cellGeo = new THREE.PlaneGeometry(0.38, 0.38); + const cellMat = new THREE.MeshBasicMaterial({ color: 0x32b8c6, transparent: true, opacity: 0.25, side: THREE.DoubleSide }); + const sigGrid = new THREE.InstancedMesh(cellGeo, cellMat, GS * GS); + sigGrid.rotation.x = -Math.PI / 2; sigGrid.position.y = 0.005; + const dum = new THREE.Object3D(); + for (let z = 0; z < GS; z++) for (let x = 0; x < GS; x++) { + dum.position.set((x - GS / 2) * 0.4, (z - GS / 2) * 0.4, 0); + dum.updateMatrix(); + sigGrid.setMatrixAt(z * GS + x, dum.matrix); + sigGrid.setColorAt(z * GS + x, new THREE.Color(0x080c16)); + } + sigGrid.instanceMatrix.needsUpdate = true; + if (sigGrid.instanceColor) sigGrid.instanceColor.needsUpdate = true; + scene.add(sigGrid); + + // --- ESP32 nodes --- + const nodeGeo = new THREE.OctahedronGeometry(0.08, 1); + const nodeMs: THREE.Mesh[] = []; + for (let i = 0; i < 8; i++) { + const mat = new THREE.MeshStandardMaterial({ color: 0x00ff88, emissive: 0x00ff88, emissiveIntensity: 0.7, wireframe: true }); + const m = new THREE.Mesh(nodeGeo, mat); m.visible = false; scene.add(m); nodeMs.push(m); } - // Bone lines - const boneMat = new THREE.LineBasicMaterial({ - color: 0x32b8c6, - transparent: true, - opacity: 0.7, - }); - const boneLines = BONES.map(([a, b]) => { - const g = new THREE.BufferGeometry().setFromPoints([ - new THREE.Vector3(), - new THREE.Vector3(), - ]); - const l = new THREE.Line(g, boneMat); - l.visible = false; - scene.add(l); - return { line: l, a, b }; + // --- Human body: DensePose-colored capsule mesh --- + // Head: slightly oblate sphere + const headGeo = new THREE.SphereGeometry(0.105, 20, 16); + headGeo.scale(1, 1.08, 1); + const headMat = new THREE.MeshPhysicalMaterial({ + color: DENSEPOSE_COLORS.head, emissive: DENSEPOSE_COLORS.head, + emissiveIntensity: 0.08, roughness: 0.3, metalness: 0.05, + clearcoat: 0.4, clearcoatRoughness: 0.3, transparent: true, opacity: 0.9, }); + const headM = new THREE.Mesh(headGeo, headMat); + headM.castShadow = true; headM.visible = false; scene.add(headM); - // Particle field - const N = 500; - const particleGeo = new THREE.BufferGeometry(); - const pPos = new Float32Array(N * 3); - for (let i = 0; i < N; i++) { - pPos[i * 3] = (Math.random() - 0.5) * 16; - pPos[i * 3 + 1] = Math.random() * 4; - pPos[i * 3 + 2] = (Math.random() - 0.5) * 16; + // Head glow + const headGlowGeo = new THREE.SphereGeometry(0.14, 12, 10); + const headGlowMat = new THREE.MeshBasicMaterial({ + color: DENSEPOSE_COLORS.head, transparent: true, opacity: 0.08, side: THREE.BackSide, + }); + const headGlowM = new THREE.Mesh(headGlowGeo, headGlowMat); + headGlowM.visible = false; scene.add(headGlowM); + + // Eyes + const eyeGeo = new THREE.SphereGeometry(0.015, 8, 6); + const eyeMat = new THREE.MeshBasicMaterial({ color: 0xeeffff }); + const eyeL = new THREE.Mesh(eyeGeo, eyeMat); + const eyeR = new THREE.Mesh(eyeGeo, eyeMat.clone()); + eyeL.visible = eyeR.visible = false; + scene.add(eyeL); scene.add(eyeR); + + // Pupils + const pupilGeo = new THREE.SphereGeometry(0.008, 6, 4); + const pupilMat = new THREE.MeshBasicMaterial({ color: 0x112233 }); + const pupilL = new THREE.Mesh(pupilGeo, pupilMat); + const pupilR = new THREE.Mesh(pupilGeo, pupilMat.clone()); + pupilL.visible = pupilR.visible = false; + scene.add(pupilL); scene.add(pupilR); + + // Neck + const neckGeo = new THREE.CapsuleGeometry(0.04, 0.08, 4, 8); + const neckMat = new THREE.MeshPhysicalMaterial({ + color: DENSEPOSE_COLORS.neck, emissive: DENSEPOSE_COLORS.neck, + emissiveIntensity: 0.05, roughness: 0.4, transparent: true, opacity: 0.85, + }); + const neckM = new THREE.Mesh(neckGeo, neckMat); + neckM.castShadow = true; neckM.visible = false; scene.add(neckM); + + // Torso: front plate + const torsoGeo = new THREE.BoxGeometry(0.34, 0.50, 0.18, 2, 3, 2); + // Round the torso vertices slightly + const torsoPos = torsoGeo.attributes.position; + for (let i = 0; i < torsoPos.count; i++) { + const x = torsoPos.getX(i), y = torsoPos.getY(i), z = torsoPos.getZ(i); + const r = Math.sqrt(x * x + z * z); + if (r > 0.01) { + const bulge = 1 + 0.15 * Math.cos(y * 3.5); // chest & hip curvature + torsoPos.setX(i, x * bulge); + torsoPos.setZ(i, z * bulge); + } } - particleGeo.setAttribute('position', new THREE.BufferAttribute(pPos, 3)); - const pMat = new THREE.PointsMaterial({ - color: 0x32b8c6, - size: 0.04, - transparent: true, - opacity: 0.4, + torsoGeo.computeVertexNormals(); + const torsoMat = new THREE.MeshPhysicalMaterial({ + color: DENSEPOSE_COLORS.torsoFront, emissive: DENSEPOSE_COLORS.torsoFront, + emissiveIntensity: 0.06, roughness: 0.35, metalness: 0.05, + clearcoat: 0.2, transparent: true, opacity: 0.88, }); - scene.add(new THREE.Points(particleGeo, pMat)); + const torsoM = new THREE.Mesh(torsoGeo, torsoMat); + torsoM.castShadow = true; torsoM.visible = false; scene.add(torsoM); - // Signal ring - const ringGeo = new THREE.TorusGeometry(2, 0.02, 8, 64); - const ringMat = new THREE.MeshBasicMaterial({ - color: 0x32b8c6, - transparent: true, - opacity: 0.3, + // Torso glow + const torsoGlowGeo = new THREE.BoxGeometry(0.40, 0.55, 0.24); + const torsoGlowMat = new THREE.MeshBasicMaterial({ + color: DENSEPOSE_COLORS.torsoFront, transparent: true, opacity: 0.06, side: THREE.BackSide, }); - const ring = new THREE.Mesh(ringGeo, ringMat); - ring.rotation.x = Math.PI / 2; - ring.position.y = 0.01; - scene.add(ring); + const torsoGlowM = new THREE.Mesh(torsoGlowGeo, torsoGlowMat); + torsoGlowM.visible = false; scene.add(torsoGlowM); + + // Hands (small boxes) + const handGeo = new THREE.BoxGeometry(0.05, 0.08, 0.025, 1, 1, 1); + const handLMat = new THREE.MeshPhysicalMaterial({ color: DENSEPOSE_COLORS.lHand, emissive: DENSEPOSE_COLORS.lHand, emissiveIntensity: 0.1, roughness: 0.3, transparent: true, opacity: 0.85 }); + const handRMat = new THREE.MeshPhysicalMaterial({ color: DENSEPOSE_COLORS.rHand, emissive: DENSEPOSE_COLORS.rHand, emissiveIntensity: 0.1, roughness: 0.3, transparent: true, opacity: 0.85 }); + const handL = new THREE.Mesh(handGeo, handLMat); handL.visible = false; scene.add(handL); + const handR = new THREE.Mesh(handGeo, handRMat); handR.visible = false; scene.add(handR); + + // Feet (wedge-like boxes) + const footGeo = new THREE.BoxGeometry(0.06, 0.04, 0.14, 1, 1, 1); + const footLMat = new THREE.MeshPhysicalMaterial({ color: DENSEPOSE_COLORS.lFoot, emissive: DENSEPOSE_COLORS.lFoot, emissiveIntensity: 0.1, roughness: 0.4, transparent: true, opacity: 0.85 }); + const footRMat = new THREE.MeshPhysicalMaterial({ color: DENSEPOSE_COLORS.rFoot, emissive: DENSEPOSE_COLORS.rFoot, emissiveIntensity: 0.1, roughness: 0.4, transparent: true, opacity: 0.85 }); + const footL = new THREE.Mesh(footGeo, footLMat); footL.visible = false; scene.add(footL); + const footR = new THREE.Mesh(footGeo, footRMat); footR.visible = false; scene.add(footR); + + // Limb capsules + glow capsules + const limbMs = BODY_SEGS.map(([,, rT, rB, ck]) => makePart(scene, rT, rB, DENSEPOSE_COLORS[ck])); + const limbGlowMs = BODY_SEGS.map(([,, rT, rB, ck]) => makePart(scene, rT * 1.6, rB * 1.6, DENSEPOSE_COLORS[ck], true)); + + // Joint dots + const jDotGeo = new THREE.SphereGeometry(0.018, 6, 4); + const jDots = Array.from({ length: 17 }, () => { + const mat = new THREE.MeshBasicMaterial({ color: 0x88ddee, transparent: true, opacity: 0.7 }); + const m = new THREE.Mesh(jDotGeo, mat); m.visible = false; scene.add(m); return m; + }); + + // Skeleton lines (thin wireframe overlay) + const skelMat = new THREE.LineBasicMaterial({ color: 0x55ccdd, transparent: true, opacity: 0.25 }); + const skelLines = BONES.map(([a, b]) => { + const g = new THREE.BufferGeometry().setFromPoints([new THREE.Vector3(), new THREE.Vector3()]); + const l = new THREE.Line(g, skelMat); l.visible = false; scene.add(l); return { line: l, a, b }; + }); + + // Heart ring + const hrGeo = new THREE.TorusGeometry(0.18, 0.006, 8, 32); + const hrMat = new THREE.MeshBasicMaterial({ color: 0xff3355, transparent: true, opacity: 0 }); + const hrRing = new THREE.Mesh(hrGeo, hrMat); hrRing.visible = false; scene.add(hrRing); + + // Breathing indicator rings (concentric around chest) + const brRings = [0.22, 0.28, 0.34].map((r) => { + const geo = new THREE.TorusGeometry(r, 0.003, 6, 32); + const mat = new THREE.MeshBasicMaterial({ color: 0x44ddaa, transparent: true, opacity: 0 }); + const m = new THREE.Mesh(geo, mat); m.visible = false; scene.add(m); return m; + }); + + // WiFi pulse rings + const wifiRings = [1.0, 1.8, 2.6].map((r) => { + const geo = new THREE.TorusGeometry(r, 0.01, 6, 48); + const mat = new THREE.MeshBasicMaterial({ color: 0x32b8c6, transparent: true, opacity: 0.15 }); + const m = new THREE.Mesh(geo, mat); m.rotation.x = Math.PI / 2; m.position.y = 0.01; scene.add(m); return m; + }); + + // Particles + const NP = 400; + const pGeo = new THREE.BufferGeometry(); + const pA = new Float32Array(NP * 3); + for (let i = 0; i < NP; i++) { + pA[i * 3] = (Math.random() - 0.5) * 12; + pA[i * 3 + 1] = Math.random() * 3.5; + pA[i * 3 + 2] = (Math.random() - 0.5) * 12; + } + pGeo.setAttribute('position', new THREE.BufferAttribute(pA, 3)); + scene.add(new THREE.Points(pGeo, new THREE.PointsMaterial({ + color: 0x3399bb, size: 0.018, transparent: true, opacity: 0.25, + }))); + + // --- HUD --- + const hudC = document.createElement('canvas'); hudC.width = 640; hudC.height = 128; + const hudT = new THREE.CanvasTexture(hudC); + const hudS = new THREE.Sprite(new THREE.SpriteMaterial({ map: hudT, transparent: true })); + hudS.scale.set(3.2, 0.64, 1); hudS.position.set(0, 3.2, 0); scene.add(hudS); + + // --- Smooth keypoints --- + const smoothKps: THREE.Vector3[] = BASE_POSE.map(([x, y, z]) => new THREE.Vector3(x, y, z)); + const targetKps: THREE.Vector3[] = BASE_POSE.map(([x, y, z]) => new THREE.Vector3(x, y, z)); + const tmpA = new THREE.Vector3(); + const tmpB = new THREE.Vector3(); + const hc = new THREE.Color(); // State - const state = { - renderer, - scene, - camera, - joints, - boneLines, - ring, - particleGeo, - pointLight, - animId: 0, - cameraAngle: 0, - cameraRadius: 6, - cameraY: 2, - isDragging: false, - frameCount: 0, - lastFpsTime: performance.now(), + const state: any = { + renderer, scene, camera, animId: 0, + camAngle: 0, camR: 3.5, camY: 1.4, + drag: false, fCount: 0, fpsT: performance.now(), + prevPresence: false, fadeIn: 0, }; sceneRef.current = state; - // Mouse interaction - const canvas = renderer.domElement; - const onMouseDown = () => { state.isDragging = true; }; - const onMouseUp = () => { state.isDragging = false; }; - const onMouseMove = (e: MouseEvent) => { - if (state.isDragging) { - state.cameraAngle += e.movementX * 0.01; - state.cameraY = Math.max(0.5, Math.min(5, state.cameraY - e.movementY * 0.01)); - } - }; - const onWheel = (e: WheelEvent) => { - state.cameraRadius = Math.max(2, Math.min(15, state.cameraRadius + e.deltaY * 0.005)); - }; - canvas.addEventListener('mousedown', onMouseDown); - canvas.addEventListener('mouseup', onMouseUp); - canvas.addEventListener('mousemove', onMouseMove); - canvas.addEventListener('wheel', onWheel, { passive: true }); + // Input + const cvs = renderer.domElement; + cvs.addEventListener('mousedown', () => { state.drag = true; }); + cvs.addEventListener('mouseup', () => { state.drag = false; }); + cvs.addEventListener('mouseleave', () => { state.drag = false; }); + cvs.addEventListener('mousemove', (e: MouseEvent) => { + if (state.drag) { state.camAngle += e.movementX * 0.006; state.camY = Math.max(0.2, Math.min(4, state.camY - e.movementY * 0.006)); } + }); + cvs.addEventListener('wheel', (e: WheelEvent) => { + state.camR = Math.max(1.5, Math.min(10, state.camR + e.deltaY * 0.003)); + }, { passive: true }); + const onR = () => { camera.aspect = W() / H(); camera.updateProjectionMatrix(); renderer.setSize(W(), H()); }; + window.addEventListener('resize', onR); - // Resize - const onResize = () => { - camera.aspect = W() / H(); - camera.updateProjectionMatrix(); - renderer.setSize(W(), H()); - }; - window.addEventListener('resize', onResize); - - // Animation loop + // --- Animate --- const animate = () => { state.animId = requestAnimationFrame(animate); const t = performance.now() * 0.001; + const fr = frameRef.current; - // Camera orbit - if (!state.isDragging) state.cameraAngle += 0.002; - camera.position.set( - Math.sin(state.cameraAngle) * state.cameraRadius, - state.cameraY, - Math.cos(state.cameraAngle) * state.cameraRadius, - ); - camera.lookAt(0, 1, 0); + // Camera + if (!state.drag) state.camAngle += 0.001; + camera.position.set(Math.sin(state.camAngle) * state.camR, state.camY, Math.cos(state.camAngle) * state.camR); + camera.lookAt(0, 0.95, 0); - // Animate ring - ring.material.opacity = 0.15 + Math.sin(t * 2) * 0.1; - const scale = 1 + Math.sin(t) * 0.1; - ring.scale.set(scale, scale, 1); + const pres = fr?.classification?.presence ?? false; + const mot = fr?.classification?.motion_level ?? 'absent'; + const conf = fr?.classification?.confidence ?? 0; + const mPow = fr?.features?.motion_band_power ?? 0; + const bPow = fr?.features?.breathing_band_power ?? 0; + const rssi = fr?.features?.mean_rssi ?? -80; - // Animate particles - const pp = particleGeo.attributes.position as THREE.BufferAttribute; - for (let i = 0; i < N; i++) { - (pp.array as Float32Array)[i * 3 + 1] += Math.sin(t + i) * 0.001; + // Fade body in/out (gradual transitions) + if (pres && conf > 0.2) state.fadeIn = Math.min(1, state.fadeIn + 0.015); + else state.fadeIn = Math.max(0, state.fadeIn - 0.008); + const show = state.fadeIn > 0.01; + const alpha = state.fadeIn; + + // --- Compute target keypoints --- + for (let i = 0; i < 17; i++) { + const [bx, by, bz] = BASE_POSE[i]; + let ax = bx, ay = by, az = bz; + + if (pres) { + // Breathing: gentle chest rise/fall + const bFreq = 0.25 + bPow * 0.5; // ~15 bpm base + const bAmp = 0.004 + bPow * 0.008; + const bPhase = Math.sin(t * bFreq * Math.PI * 2); + if (i >= 5 && i <= 10) { ay += bPhase * bAmp; } + if (i <= 4) ay += bPhase * bAmp * 0.3; + + // Very subtle sway + ax += Math.sin(t * 0.35) * 0.004; + az += Math.cos(t * 0.25) * 0.002; + + if (mot === 'active') { + const ws = 1.8 + mPow * 2; + const wa = 0.03 + mPow * 0.06; + const ph = t * ws; + + // Legs + if (i === 13) { az += Math.sin(ph) * wa * 0.7; ay -= Math.abs(Math.sin(ph)) * 0.015; } + if (i === 14) { az += Math.sin(ph + Math.PI) * wa * 0.7; ay -= Math.abs(Math.sin(ph + Math.PI)) * 0.015; } + if (i === 15) { az += Math.sin(ph - 0.2) * wa * 0.8; } + if (i === 16) { az += Math.sin(ph + Math.PI - 0.2) * wa * 0.8; } + + // Arms counter-swing (subtle) + if (i === 7) az += Math.sin(ph + Math.PI) * wa * 0.35; + if (i === 8) az += Math.sin(ph) * wa * 0.35; + if (i === 9) az += Math.sin(ph + Math.PI) * wa * 0.45; + if (i === 10) az += Math.sin(ph) * wa * 0.45; + + // Tiny vertical bob + ay += Math.abs(Math.sin(ph)) * 0.006; + + } else if (mot === 'present_still') { + const it = t * 0.25; + // Very subtle weight shift + if (i >= 11) ax += Math.sin(it * 0.4) * 0.004; + // Barely perceptible hand drift + if (i === 9) { ax += Math.sin(it * 0.8) * 0.005; } + if (i === 10) { ax += Math.sin(it * 0.6 + 0.5) * 0.005; } + } + } + targetKps[i].set(ax, ay, az); + } + + // Smooth interpolation (lower = smoother, less jumpy) + const lerpA = 0.04; + for (let i = 0; i < 17; i++) lerp3(smoothKps[i], targetKps[i], lerpA); + + // --- Head --- + headM.visible = headGlowM.visible = show; + if (show) { + tmpA.copy(smoothKps[0]).add(new THREE.Vector3(0, 0.06, 0)); + headM.position.copy(tmpA); + headGlowM.position.copy(tmpA); + (headM.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.9; + headGlowMat.opacity = alpha * 0.08; + } + + // Eyes + pupils + eyeL.visible = eyeR.visible = pupilL.visible = pupilR.visible = show; + if (show) { + const headPos = headM.position; + eyeL.position.set(headPos.x - 0.032, headPos.y + 0.01, headPos.z + 0.09); + eyeR.position.set(headPos.x + 0.032, headPos.y + 0.01, headPos.z + 0.09); + pupilL.position.set(eyeL.position.x, eyeL.position.y, eyeL.position.z + 0.012); + pupilR.position.set(eyeR.position.x, eyeR.position.y, eyeR.position.z + 0.012); + } + + // Neck + neckM.visible = show; + if (show) { + const neckTop = new THREE.Vector3().copy(smoothKps[0]).add(new THREE.Vector3(0, -0.04, 0)); + const neckBot = tmpA.addVectors(smoothKps[5], smoothKps[6]).multiplyScalar(0.5).add(new THREE.Vector3(0, 0.04, 0)); + neckM.position.addVectors(neckTop, neckBot).multiplyScalar(0.5); + neckM.scale.y = neckTop.distanceTo(neckBot) * 4; + (neckM.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.85; + } + + // Torso + torsoM.visible = torsoGlowM.visible = show; + if (show) { + const mSh = tmpA.addVectors(smoothKps[5], smoothKps[6]).multiplyScalar(0.5); + const mHp = tmpB.addVectors(smoothKps[11], smoothKps[12]).multiplyScalar(0.5); + const tPos = new THREE.Vector3().addVectors(mSh, mHp).multiplyScalar(0.5); + torsoM.position.copy(tPos); + torsoGlowM.position.copy(tPos); + const bScale = 1 + Math.sin(t * (0.9 + bPow * 4) * Math.PI * 2) * 0.02 * (1 + bPow * 3); + torsoM.scale.set(1, 1, bScale); + (torsoM.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.88; + torsoGlowMat.opacity = alpha * 0.06; + } + + // Hands + handL.visible = handR.visible = show; + if (show) { + handL.position.copy(smoothKps[9]).add(new THREE.Vector3(0, -0.04, 0)); + handR.position.copy(smoothKps[10]).add(new THREE.Vector3(0, -0.04, 0)); + (handL.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.85; + (handR.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.85; + } + + // Feet + footL.visible = footR.visible = show; + if (show) { + footL.position.copy(smoothKps[15]).add(new THREE.Vector3(0, 0.02, 0.04)); + footR.position.copy(smoothKps[16]).add(new THREE.Vector3(0, 0.02, 0.04)); + (footL.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.85; + (footR.material as THREE.MeshPhysicalMaterial).opacity = alpha * 0.85; + } + + // Limb capsules β€” emissive reacts to motion intensity + BODY_SEGS.forEach(([ai, bi, rT, rB], idx) => { + limbMs[idx].visible = limbGlowMs[idx].visible = show; + if (show) { + positionLimb(limbMs[idx], smoothKps[ai], smoothKps[bi], rT, rB); + positionLimb(limbGlowMs[idx], smoothKps[ai], smoothKps[bi], rT * 1.6, rB * 1.6); + const limbMat = limbMs[idx].material as THREE.MeshPhysicalMaterial; + limbMat.opacity = alpha * 0.82; + // Glow brighter with more motion (direct sensor feedback) + limbMat.emissiveIntensity = 0.06 + mPow * 0.4; + const glowMat = limbGlowMs[idx].material as THREE.MeshPhysicalMaterial; + glowMat.opacity = alpha * (0.06 + mPow * 0.15); + } + }); + + // Joint dots & skeleton lines + jDots.forEach((d, i) => { d.visible = show; if (show) d.position.copy(smoothKps[i]); }); + skelLines.forEach(({ line, a, b }) => { + line.visible = show; + if (show) { + const p = line.geometry.attributes.position as THREE.BufferAttribute; + p.setXYZ(0, smoothKps[a].x, smoothKps[a].y, smoothKps[a].z); + p.setXYZ(1, smoothKps[b].x, smoothKps[b].y, smoothKps[b].z); + p.needsUpdate = true; + } + }); + + // Heart ring + const vs = fr?.vital_signs as Record | undefined; + const hrBpm = Number(vs?.hr_proxy_bpm ?? vs?.heart_rate_bpm ?? 0); + hrRing.visible = show && hrBpm > 0; + if (hrRing.visible) { + const chst = tmpA.addVectors(smoothKps[5], smoothKps[6]).multiplyScalar(0.5); + chst.y -= 0.08; + hrRing.position.copy(chst); + hrRing.lookAt(camera.position); + const bp = (t * (hrBpm / 60) * Math.PI * 2) % (Math.PI * 2); + const beat = Math.pow(Math.max(0, Math.sin(bp)), 10); + hrMat.opacity = beat * 0.5 * alpha; + hrRing.scale.setScalar(1 + beat * 0.12); + } + + // Breathing rings + brRings.forEach((ring, ri) => { + ring.visible = show && bPow > 0.01; + if (ring.visible) { + const chst = tmpA.addVectors(smoothKps[5], smoothKps[6]).multiplyScalar(0.5); + chst.y -= 0.05; + ring.position.copy(chst); + ring.lookAt(camera.position); + const bph = Math.sin(t * (0.9 + bPow * 4) * Math.PI * 2 - ri * 0.5); + (ring.material as THREE.MeshBasicMaterial).opacity = Math.max(0, bph * 0.2 * alpha); + ring.scale.setScalar(1 + bph * 0.08); + } + }); + + // WiFi pulse rings + wifiRings.forEach((wr, wi) => { + const phase = (t * 0.5 + wi * 0.4) % 1; + wr.scale.setScalar(0.8 + phase * 1.5 + mPow); + (wr.material as THREE.MeshBasicMaterial).opacity = (1 - phase) * 0.12 * (pres ? 1 : 0.3); + }); + + // ESP32 nodes + (fr?.nodes || []).forEach((n, i) => { + if (i < nodeMs.length) { + const [px, py, pz] = n.position; + nodeMs[i].position.set(px * 2, py + 0.12, pz * 2); + nodeMs[i].visible = true; nodeMs[i].rotation.y = t * 0.4 + i; + (nodeMs[i].material as THREE.MeshStandardMaterial).emissiveIntensity = 0.5 + Math.sin(t * 3 + i) * 0.3; + } + }); + for (let i = (fr?.nodes || []).length; i < nodeMs.length; i++) nodeMs[i].visible = false; + + // Signal field + const sf = fr?.signal_field; + if (sf?.values?.length) { + const gx = sf.grid_size[0], gz = sf.grid_size[2]; + for (let zi = 0; zi < Math.min(gz, GS); zi++) for (let xi = 0; xi < Math.min(gx, GS); xi++) { + const v = sf.values[zi * gx + xi] || 0; + if (v < 0.25) hc.setRGB(0.03, 0.05 + v * 1.8, 0.08 + v * 1.8); + else if (v < 0.5) hc.setRGB(0.03, 0.2 + (v - 0.25) * 2.4, 0.5 - (v - 0.25) * 1.2); + else if (v < 0.75) hc.setRGB((v - 0.5) * 4, 0.7 + (v - 0.5) * 0.6, 0.1); + else hc.setRGB(1, 1 - (v - 0.75) * 3, 0.05); + sigGrid.setColorAt(zi * GS + xi, hc); + } + if (sigGrid.instanceColor) sigGrid.instanceColor.needsUpdate = true; + } + + // Lighting follows data + rim.intensity = 0.8 + Math.abs(rssi + 50) * 0.015; + + // Particles + const pp = pGeo.attributes.position as THREE.BufferAttribute; + for (let i = 0; i < NP; i++) { + (pp.array as Float32Array)[i * 3 + 1] += Math.sin(t * 0.8 + i * 0.5) * 0.0006 + mPow * 0.001; + if ((pp.array as Float32Array)[i * 3 + 1] > 3.5) (pp.array as Float32Array)[i * 3 + 1] = 0; } pp.needsUpdate = true; - // Update skeleton from frame data - const currentFrame = frameRef.current; - if (currentFrame) { - const persons = (currentFrame as any).persons || []; - if (persons.length > 0) { - const kps = persons[0].keypoints || []; - kps.forEach((kp: any, i: number) => { - if (i < 17 && joints[i]) { - joints[i].position.set( - (kp.x - 0.5) * 4, - (1 - kp.y) * 3, - (kp.z || 0) * 2, - ); - joints[i].visible = kp.confidence > 0.3; - (joints[i].material as THREE.MeshStandardMaterial).emissiveIntensity = - 0.3 + kp.confidence * 0.7; - } - }); - boneLines.forEach(({ line, a, b }) => { - if (joints[a].visible && joints[b].visible) { - const pos = line.geometry.attributes.position as THREE.BufferAttribute; - pos.setXYZ(0, joints[a].position.x, joints[a].position.y, joints[a].position.z); - pos.setXYZ(1, joints[b].position.x, joints[b].position.y, joints[b].position.z); - pos.needsUpdate = true; - line.visible = true; - } else { - line.visible = false; - } - }); - } else { - joints.forEach((j) => { j.visible = false; }); - boneLines.forEach((bl) => { bl.line.visible = false; }); + // HUD + const ctx = hudC.getContext('2d'); + if (ctx && fr) { + ctx.clearRect(0, 0, 640, 128); + ctx.font = 'bold 14px "SF Mono", Menlo, monospace'; + ctx.fillStyle = '#32b8c6'; + ctx.fillText(`WIFI-DENSEPOSE [${(fr.source || '--').toUpperCase()}]`, 12, 20); + ctx.font = '12px "SF Mono", Menlo, monospace'; + ctx.fillStyle = '#7799aa'; + ctx.fillText(`Nodes: ${(fr.nodes || []).length} RSSI: ${rssi.toFixed(1)} dBm Motion: ${mot} Conf: ${(conf * 100).toFixed(0)}%`, 12, 42); + if (vs) { + const br = Number(vs.breathing_bpm ?? vs.breathing_rate_bpm ?? 0); + if (br > 0 || hrBpm > 0) { + ctx.fillStyle = '#44ddaa'; + ctx.fillText(`Breathing: ${br.toFixed(1)} bpm Heart: ${hrBpm.toFixed(1)} bpm`, 12, 62); + } } - - // Adjust light from RSSI - const features = (currentFrame as any).features; - if (features) { - const rssi = features.mean_rssi || -70; - pointLight.intensity = 1 + Math.abs(rssi + 50) * 0.02; + if (show) { + ctx.fillStyle = pres ? (mot === 'active' ? '#ff8844' : '#44bbcc') : '#556677'; + const mBar = Math.min(20, Math.round(mPow * 40)); + const mBarStr = '\u2588'.repeat(mBar) + '\u2591'.repeat(20 - mBar); + ctx.fillText(`Motion: [${mBarStr}] ${(mPow * 100).toFixed(0)}%`, 12, 82); + ctx.fillStyle = '#556677'; + ctx.font = '10px "SF Mono", Menlo, monospace'; + ctx.fillText('Pose: procedural (load NN model for limb tracking)', 12, 100); } + hudT.needsUpdate = true; } renderer.render(scene, camera); - // FPS counter - state.frameCount++; - if (performance.now() - state.lastFpsTime >= 1000) { - onFps(state.frameCount); - state.frameCount = 0; - state.lastFpsTime = performance.now(); + state.fCount++; + if (performance.now() - state.fpsT >= 1000) { + onFps(state.fCount); state.fCount = 0; state.fpsT = performance.now(); } }; @@ -280,15 +678,10 @@ export const GaussianSplatWebViewWeb = ({ onReady, onFps, onError, frame }: Gaus onReady(); return () => { - canvas.removeEventListener('mousedown', onMouseDown); - canvas.removeEventListener('mouseup', onMouseUp); - canvas.removeEventListener('mousemove', onMouseMove); - canvas.removeEventListener('wheel', onWheel); - window.removeEventListener('resize', onResize); + cvs.removeEventListener('mousedown', () => {}); + window.removeEventListener('resize', onR); cleanup(); - if (container.contains(renderer.domElement)) { - container.removeChild(renderer.domElement); - } + if (container.contains(renderer.domElement)) container.removeChild(renderer.domElement); }; } catch (err) { onError(err instanceof Error ? err.message : 'Failed to initialize 3D renderer'); @@ -298,19 +691,10 @@ export const GaussianSplatWebViewWeb = ({ onReady, onFps, onError, frame }: Gaus return ( -
+
); }; -const styles = StyleSheet.create({ - container: { - flex: 1, - backgroundColor: '#0a0e1a', - }, -}); - +const styles = StyleSheet.create({ container: { flex: 1, backgroundColor: '#080c16' } }); export default GaussianSplatWebViewWeb; diff --git a/ui/mobile/src/types/sensing.ts b/ui/mobile/src/types/sensing.ts index e9e4268c..0201c3ba 100644 --- a/ui/mobile/src/types/sensing.ts +++ b/ui/mobile/src/types/sensing.ts @@ -32,11 +32,30 @@ export interface SignalField { } export interface VitalsData { - breathing_bpm: number; - hr_proxy_bpm: number; + breathing_bpm?: number; + hr_proxy_bpm?: number; + // Rust sensing server uses these field names + breathing_rate_bpm?: number; + breathing_confidence?: number; + heart_rate_bpm?: number; + heart_confidence?: number; + confidence?: number; +} + +export interface PoseKeypoint { + name?: string; + x: number; + y: number; + z: number; confidence: number; } +export interface PersonDetection { + id?: number; + confidence: number; + keypoints: PoseKeypoint[]; +} + export interface SensingFrame { type?: string; timestamp?: number; @@ -47,4 +66,8 @@ export interface SensingFrame { classification: Classification; signal_field: SignalField; vital_signs?: VitalsData; + pose_keypoints?: [number, number, number, number][]; + persons?: PersonDetection[]; + posture?: string; + signal_quality_score?: number; } diff --git a/ui/services/pose.service.js b/ui/services/pose.service.js index f5e06269..072fc5f4 100644 --- a/ui/services/pose.service.js +++ b/ui/services/pose.service.js @@ -511,6 +511,7 @@ export class PoseService { persons: persons, zone_summary: zoneSummary, processing_time_ms: zoneData.metadata?.processing_time_ms || 0, + pose_source: originalMessage.pose_source || zoneData.pose_source || null, metadata: { mock_data: false, source: 'websocket', diff --git a/ui/services/sensing.service.js b/ui/services/sensing.service.js index d789a3c7..cd6a1ccd 100644 --- a/ui/services/sensing.service.js +++ b/ui/services/sensing.service.js @@ -4,8 +4,9 @@ * Manages the connection to the Python sensing WebSocket server * (ws://localhost:8765) and provides a callback-based API for the UI. * - * Falls back to simulated data if the server is unreachable so the UI - * always shows something. + * Falls back to simulated data only after MAX_RECONNECT_ATTEMPTS exhausted. + * While reconnecting the service stays in "reconnecting" state and does NOT + * emit simulated frames so the UI can clearly distinguish live vs. fallback data. */ // Derive WebSocket URL from the page origin so it works on any port @@ -14,7 +15,10 @@ const _wsProto = (typeof window !== 'undefined' && window.location.protocol === const _wsHost = (typeof window !== 'undefined' && window.location.host) ? window.location.host : 'localhost:3000'; const SENSING_WS_URL = `${_wsProto}//${_wsHost}/ws/sensing`; const RECONNECT_DELAYS = [1000, 2000, 4000, 8000, 16000]; -const MAX_RECONNECT_ATTEMPTS = 10; +const MAX_RECONNECT_ATTEMPTS = 20; +// Number of failed attempts that must occur before simulation starts. +// This prevents the UI from flashing "SIMULATED" on a brief hiccup. +const SIM_FALLBACK_AFTER_ATTEMPTS = 5; const SIMULATION_INTERVAL = 500; // ms class SensingService { @@ -26,7 +30,10 @@ class SensingService { this._reconnectAttempt = 0; this._reconnectTimer = null; this._simTimer = null; - this._state = 'disconnected'; // disconnected | connecting | connected | simulated + // Connection state: disconnected | connecting | connected | reconnecting | simulated + this._state = 'disconnected'; + // Data-source label exposed to the UI: "live" | "reconnecting" | "simulated" + this._dataSource = 'reconnecting'; this._lastMessage = null; // Ring buffer of recent RSSI values for sparkline @@ -76,6 +83,16 @@ class SensingService { return this._state; } + /** + * Current data source label. + * "live" β€” frames are arriving from the real ESP32 over WebSocket + * "reconnecting" β€” WebSocket disconnected; actively retrying, no frames emitted + * "simulated" β€” max reconnect attempts exhausted; emitting synthetic frames + */ + get dataSource() { + return this._dataSource; + } + // ---- Connection -------------------------------------------------------- _connect() { @@ -96,6 +113,7 @@ class SensingService { this._reconnectAttempt = 0; this._stopSimulation(); this._setState('connected'); + this._setDataSource('live'); }; this._ws.onmessage = (evt) => { @@ -118,28 +136,33 @@ class SensingService { this._scheduleReconnect(); } else { this._setState('disconnected'); + this._setDataSource('reconnecting'); } }; } _scheduleReconnect() { if (this._reconnectAttempt >= MAX_RECONNECT_ATTEMPTS) { - console.warn('[Sensing] Max reconnect attempts reached, switching to simulation'); + console.warn('[Sensing] Max reconnect attempts (%d) reached, switching to simulation', MAX_RECONNECT_ATTEMPTS); this._fallbackToSimulation(); return; } const delay = RECONNECT_DELAYS[Math.min(this._reconnectAttempt, RECONNECT_DELAYS.length - 1)]; this._reconnectAttempt++; - console.info('[Sensing] Reconnecting in %dms (attempt %d)', delay, this._reconnectAttempt); + console.info('[Sensing] Reconnecting in %dms (attempt %d/%d)', delay, this._reconnectAttempt, MAX_RECONNECT_ATTEMPTS); + + this._setState('reconnecting'); + this._setDataSource('reconnecting'); this._reconnectTimer = setTimeout(() => { this._reconnectTimer = null; this._connect(); }, delay); - // Start simulation while waiting - if (this._state !== 'simulated') { + // Only start simulation after several failed attempts so a brief hiccup + // does not immediately switch the UI to "SIMULATED DATA". + if (this._reconnectAttempt >= SIM_FALLBACK_AFTER_ATTEMPTS && this._state !== 'simulated') { this._fallbackToSimulation(); } } @@ -148,6 +171,7 @@ class SensingService { _fallbackToSimulation() { this._setState('simulated'); + this._setDataSource('simulated'); if (this._simTimer) return; // already running console.info('[Sensing] Running in simulation mode'); @@ -196,6 +220,9 @@ class SensingService { type: 'sensing_update', timestamp: t, source: 'simulated', + // Explicit machine-readable marker so the UI can always detect simulated + // frames regardless of which code path produced them. + _simulated: true, nodes: [{ node_id: 1, rssi_dbm: baseRssi + Math.sin(t * 0.5) * 3, @@ -262,6 +289,21 @@ class SensingService { } } + /** + * Update the dataSource label and notify state listeners so the UI can + * react without needing a separate subscription. + * @param {'live'|'reconnecting'|'simulated'} source + */ + _setDataSource(source) { + if (source === this._dataSource) return; + this._dataSource = source; + // Re-use the same state-listener channel β€” listeners receive the + // connection state but can read dataSource via service.dataSource. + for (const cb of this._stateListeners) { + try { cb(this._state); } catch (e) { /* ignore */ } + } + } + _clearTimers() { this._stopSimulation(); if (this._reconnectTimer) { diff --git a/ui/style.css b/ui/style.css index 38671bbd..76c8cf60 100644 --- a/ui/style.css +++ b/ui/style.css @@ -1754,6 +1754,11 @@ canvas { background: var(--color-error); } +.sensing-dot.reconnecting { + background: var(--color-warning); + animation: pulse 1.5s infinite; +} + .sensing-source { margin-left: auto; font-size: var(--font-size-xs); @@ -1761,6 +1766,52 @@ canvas { font-family: var(--font-family-mono); } +.sensing-about-text { + margin: 0; + font-size: 12px; + color: #aaa; + line-height: 1.5; +} + +.sensing-about-text strong { + color: #ccc; +} + +/* Data-source status banner (live / reconnecting / simulated) */ +.sensing-source-banner { + display: block; + width: 100%; + padding: var(--space-8) var(--space-12); + margin-bottom: var(--space-12); + border-radius: var(--radius-md); + font-size: var(--font-size-sm); + font-weight: var(--font-weight-semibold); + font-family: var(--font-family-mono); + text-align: center; + letter-spacing: 0.06em; + text-transform: uppercase; + box-sizing: border-box; +} + +.sensing-source-live { + background: rgba(0, 204, 136, 0.15); + border: 1px solid #00cc88; + color: #00cc88; +} + +.sensing-source-reconnecting { + background: rgba(255, 180, 0, 0.12); + border: 1px solid var(--color-warning); + color: var(--color-warning); + animation: pulse 1.5s infinite; +} + +.sensing-source-simulated { + background: rgba(255, 60, 60, 0.12); + border: 1px solid var(--color-error); + color: var(--color-error); +} + /* Big RSSI value */ .sensing-big-value { font-size: var(--font-size-3xl); diff --git a/ui/utils/pose-renderer.js b/ui/utils/pose-renderer.js index efcefc22..6ab8c214 100644 --- a/ui/utils/pose-renderer.js +++ b/ui/utils/pose-renderer.js @@ -183,31 +183,132 @@ export class PoseRenderer { } } - // Keypoints only mode + // Keypoints only mode β€” large colored dots with labels, no skeleton lines renderKeypointsMode(poseData, metadata) { const persons = poseData.persons || []; - + persons.forEach((person, index) => { if (person.confidence >= this.config.confidenceThreshold && person.keypoints) { this.renderKeypoints(person.keypoints, person.confidence, true); + + // Render bounding box + if (this.config.showBoundingBox && person.bbox) { + this.renderBoundingBox(person.bbox, person.confidence, index); + } + if (this.config.showConfidence) { + this.renderConfidenceScore(person, index); + } } }); + + if (this.config.showZones && poseData.zone_summary) { + this.renderZones(poseData.zone_summary); + } } - // Heatmap rendering mode + // Heatmap rendering mode β€” Gaussian blobs around each keypoint renderHeatmapMode(poseData, metadata) { - // This would render a heatmap visualization - // For now, fall back to skeleton mode - this.logger.debug('Heatmap mode not fully implemented, using skeleton mode'); - this.renderSkeletonMode(poseData, metadata); + const persons = poseData.persons || []; + + persons.forEach((person, personIdx) => { + if (person.confidence < this.config.confidenceThreshold || !person.keypoints) return; + + const hue = (personIdx * 60) % 360; // different hue per person + + person.keypoints.forEach((kp) => { + if (kp.confidence <= this.config.keypointConfidenceThreshold) return; + + const cx = this.scaleX(kp.x); + const cy = this.scaleY(kp.y); + const radius = 30 + kp.confidence * 20; + + const grad = this.ctx.createRadialGradient(cx, cy, 0, cx, cy, radius); + grad.addColorStop(0, `hsla(${hue}, 100%, 55%, ${kp.confidence * 0.7})`); + grad.addColorStop(0.5, `hsla(${hue}, 100%, 45%, ${kp.confidence * 0.3})`); + grad.addColorStop(1, `hsla(${hue}, 100%, 40%, 0)`); + + this.ctx.fillStyle = grad; + this.ctx.fillRect(cx - radius, cy - radius, radius * 2, radius * 2); + }); + + // Light skeleton overlay so joints are connected + if (person.keypoints) { + this.ctx.globalAlpha = 0.25; + this.renderSkeleton(person.keypoints, person.confidence); + this.ctx.globalAlpha = 1.0; + } + + if (this.config.showConfidence) { + this.renderConfidenceScore(person, personIdx); + } + }); + + if (this.config.showZones && poseData.zone_summary) { + this.renderZones(poseData.zone_summary); + } } - // Dense pose rendering mode + // Dense pose rendering mode β€” body region segmentation with filled polygons renderDenseMode(poseData, metadata) { - // This would render dense pose segmentation - // For now, fall back to skeleton mode - this.logger.debug('Dense mode not fully implemented, using skeleton mode'); - this.renderSkeletonMode(poseData, metadata); + const persons = poseData.persons || []; + + // Body part groups: [start_kp, end_kp, color] + const bodyParts = [ + { name: 'head', kps: [0, 1, 2, 3, 4], color: 'rgba(255, 100, 100, 0.4)' }, + { name: 'torso', kps: [5, 6, 12, 11], color: 'rgba(100, 200, 255, 0.4)' }, + { name: 'left_arm', kps: [5, 7, 9], color: 'rgba(100, 255, 150, 0.4)' }, + { name: 'right_arm', kps: [6, 8, 10], color: 'rgba(255, 200, 100, 0.4)' }, + { name: 'left_leg', kps: [11, 13, 15], color: 'rgba(200, 100, 255, 0.4)' }, + { name: 'right_leg', kps: [12, 14, 16], color: 'rgba(255, 255, 100, 0.4)' }, + ]; + + persons.forEach((person, personIdx) => { + if (person.confidence < this.config.confidenceThreshold || !person.keypoints) return; + + const kps = person.keypoints; + + bodyParts.forEach((part) => { + // Collect valid keypoints for this body part + const points = part.kps + .filter(i => kps[i] && kps[i].confidence > this.config.keypointConfidenceThreshold) + .map(i => ({ x: this.scaleX(kps[i].x), y: this.scaleY(kps[i].y) })); + + if (points.length < 2) return; + + // Draw filled region with padding around joints + this.ctx.fillStyle = part.color; + this.ctx.strokeStyle = part.color.replace('0.4', '0.7'); + this.ctx.lineWidth = 8; + this.ctx.lineJoin = 'round'; + this.ctx.lineCap = 'round'; + + // Draw thick path as a "region" + this.ctx.beginPath(); + this.ctx.moveTo(points[0].x, points[0].y); + for (let i = 1; i < points.length; i++) { + this.ctx.lineTo(points[i].x, points[i].y); + } + this.ctx.stroke(); + + // Draw circles at each joint to widen the region + points.forEach(p => { + this.ctx.beginPath(); + this.ctx.arc(p.x, p.y, 10, 0, Math.PI * 2); + this.ctx.fill(); + }); + }); + + // Subtle keypoint dots on top + this.renderKeypoints(kps, person.confidence, false); + + if (this.config.showConfidence) { + this.renderConfidenceScore(person, personIdx); + } + }); + + if (this.config.showZones && poseData.zone_summary) { + this.renderZones(poseData.zone_summary); + } } // Render skeleton connections