diff --git a/.gitignore b/.gitignore index 1102231d..9caaea62 100644 --- a/.gitignore +++ b/.gitignore @@ -250,3 +250,5 @@ v1/src/sensing/mac_wifi # Local build scripts firmware/esp32-csi-node/build_firmware.batdata/ models/ +demo_pointcloud.ply +demo_splats.json diff --git a/rust-port/wifi-densepose-rs/Cargo.lock b/rust-port/wifi-densepose-rs/Cargo.lock index a794fabb..962909af 100644 --- a/rust-port/wifi-densepose-rs/Cargo.lock +++ b/rust-port/wifi-densepose-rs/Cargo.lock @@ -139,6 +139,15 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "approx" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f2a05fd1bd10b2527e20a2cd32d8873d115b8b39fe219ee25f42a8aca6ba278" +dependencies = [ + "num-traits", +] + [[package]] name = "approx" version = "0.5.1" @@ -623,6 +632,27 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" +[[package]] +name = "cauchy" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ff11ddd2af3b5e80dd0297fee6e56ac038d9bdc549573cdb51bd6d2efe7f05e" +dependencies = [ + "num-complex", + "num-traits", + "rand 0.8.5", + "serde", +] + +[[package]] +name = "cblas-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6feecd82cce51b0204cf063f0041d69f24ce83f680d87514b004248e7b0fa65" +dependencies = [ + "libc", +] + [[package]] name = "cc" version = "1.2.56" @@ -1180,13 +1210,34 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys 0.4.1", +] + [[package]] name = "dirs" version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" dependencies = [ - "dirs-sys", + "dirs-sys 0.5.0", +] + +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users 0.4.6", + "windows-sys 0.48.0", ] [[package]] @@ -1197,7 +1248,7 @@ checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" dependencies = [ "libc", "option-ext", - "redox_users", + "redox_users 0.5.2", "windows-sys 0.61.2", ] @@ -1420,6 +1471,17 @@ dependencies = [ "rustc_version", ] +[[package]] +name = "filetime" +version = "0.2.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f98844151eee8917efc50bd9e8318cb963ae8b297431495d3f758616ea5c57db" +dependencies = [ + "cfg-if", + "libc", + "libredox", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" @@ -1908,7 +1970,7 @@ version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24f8647af4005fa11da47cd56252c6ef030be8fa97bdbf355e7dfb6348f0a82c" dependencies = [ - "approx", + "approx 0.5.1", "num-traits", "rstar 0.10.0", "rstar 0.11.0", @@ -2780,6 +2842,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "katexit" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccfb0b7ce7938f84a5ecbdca5d0a991e46bc9d6d078934ad5e92c5270fe547db" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "keyboard-types" version = "0.7.0" @@ -2803,6 +2876,29 @@ dependencies = [ "selectors", ] +[[package]] +name = "lapack-sys" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447f56c85fb410a7a3d36701b2153c1018b1d2b908c5fbaf01c1b04fac33bcbe" +dependencies = [ + "libc", +] + +[[package]] +name = "lax" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f96a229d9557112e574164f8024ce703625ad9f88a90964c1780809358e53da" +dependencies = [ + "cauchy", + "katexit", + "lapack-sys", + "num-traits", + "openblas-src", + "thiserror 1.0.69", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2867,7 +2963,10 @@ version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ + "bitflags 2.11.0", "libc", + "plain", + "redox_syscall 0.7.4", ] [[package]] @@ -3218,7 +3317,7 @@ version = "0.33.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26aecdf64b707efd1310e3544d709c5c0ac61c13756046aaaba41be5c4f66a3b" dependencies = [ - "approx", + "approx 0.5.1", "matrixmultiply", "nalgebra-macros", "num-complex", @@ -3271,6 +3370,9 @@ version = "0.15.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adb12d4e967ec485a5f71c6311fe28158e9d6f4bc4a447b474184d0f91a8fa32" dependencies = [ + "approx 0.4.0", + "cblas-sys", + "libc", "matrixmultiply", "num-complex", "num-integer", @@ -3310,6 +3412,22 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "ndarray-linalg" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b0e8dda0c941b64a85c5deb2b3e0144aca87aced64678adfc23eacea6d2cc42" +dependencies = [ + "cauchy", + "katexit", + "lax", + "ndarray 0.15.6", + "num-complex", + "num-traits", + "rand 0.8.5", + "thiserror 1.0.69", +] + [[package]] name = "ndarray-npy" version = "0.8.1" @@ -3441,6 +3559,8 @@ checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" dependencies = [ "bytemuck", "num-traits", + "rand 0.8.5", + "serde", ] [[package]] @@ -3670,6 +3790,32 @@ dependencies = [ "pathdiff", ] +[[package]] +name = "openblas-build" +version = "0.10.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd235aa8876fa5c4be452efde09b9b8bafa19aea0bf14a4926508213082439a3" +dependencies = [ + "anyhow", + "cc", + "flate2", + "tar", + "thiserror 2.0.18", + "ureq", +] + +[[package]] +name = "openblas-src" +version = "0.10.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fccd2c4f5271ab871f2069cb6f1a13ef2c0db50e1145ce03428ee541f4c63c4f" +dependencies = [ + "dirs 6.0.0", + "openblas-build", + "pkg-config", + "vcpkg", +] + [[package]] name = "openssl" version = "0.10.75" @@ -3819,7 +3965,7 @@ checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", - "redox_syscall", + "redox_syscall 0.5.18", "smallvec", "windows-link 0.2.1", ] @@ -4095,6 +4241,12 @@ version = "0.3.32" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +[[package]] +name = "plain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" + [[package]] name = "plist" version = "1.8.0" @@ -4694,6 +4846,26 @@ dependencies = [ "bitflags 2.11.0", ] +[[package]] +name = "redox_syscall" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +dependencies = [ + "bitflags 2.11.0", +] + +[[package]] +name = "redox_users" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba009ff324d1fc1b900bd1fdb31564febe58a8ccc8a6fdbb93b543d33b13ca43" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror 1.0.69", +] + [[package]] name = "redox_users" version = "0.5.2" @@ -5737,7 +5909,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c99284beb21666094ba2b75bbceda012e610f5479dfcc2d6e2426f53197ffd95" dependencies = [ - "approx", + "approx 0.5.1", "num-complex", "num-traits", "paste", @@ -5826,7 +5998,7 @@ dependencies = [ "objc2-foundation", "objc2-quartz-core", "raw-window-handle", - "redox_syscall", + "redox_syscall 0.5.18", "tracing", "wasm-bindgen", "web-sys", @@ -6098,6 +6270,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "tar" +version = "0.4.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22692a6476a21fa75fdfc11d452fda482af402c008cdbaf3476414e122040973" +dependencies = [ + "filetime", + "libc", + "xattr", +] + [[package]] name = "target-lexicon" version = "0.12.16" @@ -6113,7 +6296,7 @@ dependencies = [ "anyhow", "bytes", "cookie", - "dirs", + "dirs 6.0.0", "dunce", "embed_plist", "getrandom 0.3.4", @@ -6163,7 +6346,7 @@ checksum = "4bbc990d1dbf57a8e1c7fa2327f2a614d8b757805603c1b9ba5c81bade09fd4d" dependencies = [ "anyhow", "cargo_toml", - "dirs", + "dirs 6.0.0", "glob", "heck 0.5.0", "json-patch", @@ -6937,7 +7120,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5e85aa143ceb072062fc4d6356c1b520a51d636e7bc8e77ec94be3608e5e80c" dependencies = [ "crossbeam-channel", - "dirs", + "dirs 6.0.0", "libappindicator", "muda", "objc2", @@ -7673,7 +7856,7 @@ dependencies = [ name = "wifi-densepose-hardware" version = "0.3.0" dependencies = [ - "approx", + "approx 0.5.1", "byteorder", "chrono", "clap", @@ -7694,7 +7877,7 @@ name = "wifi-densepose-mat" version = "0.3.0" dependencies = [ "anyhow", - "approx", + "approx 0.5.1", "async-trait", "axum", "chrono", @@ -7743,11 +7926,26 @@ dependencies = [ "tracing", ] +[[package]] +name = "wifi-densepose-pointcloud" +version = "0.1.0" +dependencies = [ + "anyhow", + "axum", + "chrono", + "clap", + "dirs 5.0.1", + "reqwest 0.12.28", + "serde", + "serde_json", + "tokio", +] + [[package]] name = "wifi-densepose-ruvector" version = "0.3.0" dependencies = [ - "approx", + "approx 0.5.1", "criterion", "ruvector-attention 2.0.4", "ruvector-attn-mincut", @@ -7769,7 +7967,6 @@ dependencies = [ "chrono", "clap", "futures-util", - "ruvector-mincut", "serde", "serde_json", "tempfile", @@ -7777,6 +7974,7 @@ dependencies = [ "tower-http 0.5.2", "tracing", "tracing-subscriber", + "wifi-densepose-signal", "wifi-densepose-wifiscan", ] @@ -7789,6 +7987,7 @@ dependencies = [ "midstreamer-attractor", "midstreamer-temporal-compare", "ndarray 0.15.6", + "ndarray-linalg", "num-complex", "num-traits", "proptest", @@ -7808,7 +8007,7 @@ name = "wifi-densepose-train" version = "0.3.0" dependencies = [ "anyhow", - "approx", + "approx 0.5.1", "chrono", "clap", "criterion", @@ -8566,7 +8765,7 @@ dependencies = [ "block2", "cookie", "crossbeam-channel", - "dirs", + "dirs 6.0.0", "dpi", "dunce", "gdkx11", @@ -8622,6 +8821,16 @@ dependencies = [ "pkg-config", ] +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + [[package]] name = "yasna" version = "0.5.2" diff --git a/rust-port/wifi-densepose-rs/Cargo.toml b/rust-port/wifi-densepose-rs/Cargo.toml index 8245c5dd..eb917334 100644 --- a/rust-port/wifi-densepose-rs/Cargo.toml +++ b/rust-port/wifi-densepose-rs/Cargo.toml @@ -17,6 +17,7 @@ members = [ "crates/wifi-densepose-vitals", "crates/wifi-densepose-ruvector", "crates/wifi-densepose-desktop", + "crates/wifi-densepose-pointcloud", ] # ADR-040: WASM edge crate targets wasm32-unknown-unknown (no_std), # excluded from workspace to avoid breaking `cargo test --workspace`. diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/Cargo.toml b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/Cargo.toml new file mode 100644 index 00000000..a6d2700f --- /dev/null +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "wifi-densepose-pointcloud" +version = "0.1.0" +edition = "2021" +description = "Real-time dense point cloud from camera depth + WiFi CSI tomography" + +[[bin]] +name = "ruview-pointcloud" +path = "src/main.rs" + +[dependencies] +serde = { workspace = true } +serde_json = { workspace = true } +tokio = { workspace = true } +anyhow = { workspace = true } +axum = { workspace = true } +clap = { version = "4", features = ["derive"] } +chrono = "0.4" +dirs = "5" +reqwest = { version = "0.12", features = ["json"], default-features = false } diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/depth.rs b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/depth.rs new file mode 100644 index 00000000..b74d2a64 --- /dev/null +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/depth.rs @@ -0,0 +1,158 @@ +//! Monocular depth estimation via MiDaS ONNX + backprojection to 3D points. + +use crate::pointcloud::{PointCloud, ColorPoint}; +use anyhow::Result; + +/// Default camera intrinsics (approximate for HD webcam) +pub struct CameraIntrinsics { + pub fx: f32, // focal length x (pixels) + pub fy: f32, // focal length y (pixels) + pub cx: f32, // principal point x + pub cy: f32, // principal point y + pub width: u32, + pub height: u32, +} + +impl Default for CameraIntrinsics { + fn default() -> Self { + Self { + fx: 525.0, fy: 525.0, // typical webcam focal length + cx: 320.0, cy: 240.0, // center of 640x480 + width: 640, height: 480, + } + } +} + +/// Backproject a depth map to 3D points using camera intrinsics. +/// +/// depth_map: row-major [height x width] in meters +/// rgb: optional row-major [height x width x 3] color +pub fn backproject_depth( + depth_map: &[f32], + intrinsics: &CameraIntrinsics, + rgb: Option<&[u8]>, + downsample: u32, +) -> PointCloud { + let mut cloud = PointCloud::new("camera_depth"); + let w = intrinsics.width; + let h = intrinsics.height; + let step = downsample.max(1); + + for y in (0..h).step_by(step as usize) { + for x in (0..w).step_by(step as usize) { + let idx = (y * w + x) as usize; + let z = depth_map[idx]; + + // Skip invalid depths + if z <= 0.01 || z > 10.0 || z.is_nan() { continue; } + + // Backproject: (u, v, z) → (X, Y, Z) + let px = (x as f32 - intrinsics.cx) * z / intrinsics.fx; + let py = (y as f32 - intrinsics.cy) * z / intrinsics.fy; + + let (r, g, b) = if let Some(rgb_data) = rgb { + let ri = idx * 3; + if ri + 2 < rgb_data.len() { + (rgb_data[ri], rgb_data[ri + 1], rgb_data[ri + 2]) + } else { + (128, 128, 128) + } + } else { + // Color by depth (blue=near, red=far) + let t = ((z - 0.5) / 4.0).clamp(0.0, 1.0); + ((t * 255.0) as u8, ((1.0 - t) * 128.0) as u8, ((1.0 - t) * 255.0) as u8) + }; + + cloud.points.push(ColorPoint { x: px, y: py, z, r, g, b, intensity: 1.0 }); + } + } + cloud +} + +/// Run depth estimation on an image. +/// +/// When built with `--features onnx`, uses MiDaS ONNX model. +/// Otherwise, generates synthetic depth from image luminance (for testing). +pub fn estimate_depth( + image_data: &[u8], + width: u32, + height: u32, +) -> Result> { + // Luminance-based pseudo-depth (works without ONNX model) + // Darker pixels = further away (rough approximation) + let mut depth_map = vec![3.0f32; (width * height) as usize]; + for y in 0..height { + for x in 0..width { + let idx = (y * width + x) as usize; + let ri = idx * 3; + if ri + 2 < image_data.len() { + let r = image_data[ri] as f32; + let g = image_data[ri + 1] as f32; + let b = image_data[ri + 2] as f32; + let lum = (0.299 * r + 0.587 * g + 0.114 * b) / 255.0; + // Map luminance to depth: bright=near (1m), dark=far (5m) + depth_map[idx] = 1.0 + (1.0 - lum) * 4.0; + } + } + } + Ok(depth_map) +} + +/// Capture depth cloud from camera (placeholder — real impl uses nokhwa or v4l2). +pub async fn capture_depth_cloud(frames: usize) -> Result { + eprintln!("Camera capture not available (no camera on this machine)."); + eprintln!("Use --demo for synthetic data, or run on a machine with a camera."); + Ok(demo_depth_cloud()) +} + +/// Generate a demo depth point cloud (synthetic room scene). +pub fn demo_depth_cloud() -> PointCloud { + let mut cloud = PointCloud::new("demo_camera_depth"); + let intrinsics = CameraIntrinsics::default(); + + // Simulate a depth map: room with walls at 3m, floor, and a person at 2m + let w = 160; // downsampled + let h = 120; + let mut depth = vec![3.0f32; w * h]; + + // Floor plane (bottom third) + for y in (h * 2 / 3)..h { + for x in 0..w { + depth[y * w + x] = 1.0 + (y - h * 2 / 3) as f32 * 0.05; + } + } + + // Person silhouette (center, depth=2m) + for y in (h / 4)..(h * 3 / 4) { + for x in (w * 2 / 5)..(w * 3 / 5) { + let dy = (y as f32 - h as f32 / 2.0).abs() / (h as f32 / 4.0); + let dx = (x as f32 - w as f32 / 2.0).abs() / (w as f32 / 5.0); + if dx * dx + dy * dy < 1.0 { + depth[y * w + x] = 2.0 + (dx * dx + dy * dy) * 0.3; + } + } + } + + let scaled_intrinsics = CameraIntrinsics { + fx: intrinsics.fx * w as f32 / intrinsics.width as f32, + fy: intrinsics.fy * h as f32 / intrinsics.height as f32, + cx: w as f32 / 2.0, + cy: h as f32 / 2.0, + width: w as u32, + height: h as u32, + }; + + backproject_depth(&depth, &scaled_intrinsics, None, 1) +} + +fn find_midas_model() -> Result { + let paths = [ + dirs::home_dir().unwrap_or_default().join(".local/share/ruview/midas_v21_small_256.onnx"), + dirs::home_dir().unwrap_or_default().join(".cache/ruview/midas_v21_small_256.onnx"), + std::path::PathBuf::from("/usr/local/share/ruview/midas_v21_small_256.onnx"), + ]; + for p in &paths { + if p.exists() { return Ok(p.to_string_lossy().to_string()); } + } + anyhow::bail!("MiDaS ONNX model not found. Download:\n wget https://github.com/isl-org/MiDaS/releases/download/v3_1/midas_v21_small_256.onnx -O ~/.local/share/ruview/midas_v21_small_256.onnx") +} diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/fusion.rs b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/fusion.rs new file mode 100644 index 00000000..37329e8f --- /dev/null +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/fusion.rs @@ -0,0 +1,160 @@ +//! Multi-modal fusion: camera depth + WiFi RF tomography → unified point cloud. + +use crate::pointcloud::{PointCloud, ColorPoint}; +use std::collections::HashMap; + +/// Occupancy volume from WiFi RF tomography (mirrors RuView's OccupancyVolume). +#[derive(Clone, Debug)] +pub struct OccupancyVolume { + pub densities: Vec, // [nz][ny][nx] voxel densities + pub nx: usize, + pub ny: usize, + pub nz: usize, + pub bounds: [f64; 6], // [x_min, y_min, z_min, x_max, y_max, z_max] + pub occupied_count: usize, +} + +/// Convert WiFi occupancy volume to a sparse point cloud. +/// +/// Each occupied voxel (density > threshold) becomes a point at the voxel center. +pub fn occupancy_to_pointcloud(vol: &OccupancyVolume) -> PointCloud { + let mut cloud = PointCloud::new("wifi_occupancy"); + let threshold = 0.3; + + let dx = (vol.bounds[3] - vol.bounds[0]) / vol.nx as f64; + let dy = (vol.bounds[4] - vol.bounds[1]) / vol.ny as f64; + let dz = (vol.bounds[5] - vol.bounds[2]) / vol.nz as f64; + + for iz in 0..vol.nz { + for iy in 0..vol.ny { + for ix in 0..vol.nx { + let idx = iz * vol.ny * vol.nx + iy * vol.nx + ix; + let density = vol.densities[idx]; + if density > threshold { + let x = vol.bounds[0] + (ix as f64 + 0.5) * dx; + let y = vol.bounds[1] + (iy as f64 + 0.5) * dy; + let z = vol.bounds[2] + (iz as f64 + 0.5) * dz; + + // Color by density (green=low, red=high) + let t = ((density - threshold) / (1.0 - threshold)).min(1.0); + let r = (t * 255.0) as u8; + let g = ((1.0 - t) * 200.0) as u8; + + cloud.points.push(ColorPoint { + x: x as f32, + y: y as f32, + z: z as f32, + r, g, b: 50, + intensity: density as f32, + }); + } + } + } + } + cloud +} + +/// Fuse multiple point clouds with voxel-grid downsampling. +/// +/// Points from all clouds are binned into voxels of the given size. +/// Each voxel produces one averaged point (position, color, max intensity). +pub fn fuse_clouds(clouds: &[&PointCloud], voxel_size: f32) -> PointCloud { + let mut cells: HashMap<(i32, i32, i32), (f32, f32, f32, f32, f32, f32, f32, u32)> = HashMap::new(); + // (sum_x, sum_y, sum_z, sum_r, sum_g, sum_b, max_intensity, count) + + for cloud in clouds { + for p in &cloud.points { + let key = ( + (p.x / voxel_size).floor() as i32, + (p.y / voxel_size).floor() as i32, + (p.z / voxel_size).floor() as i32, + ); + let entry = cells.entry(key).or_insert((0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0)); + entry.0 += p.x; + entry.1 += p.y; + entry.2 += p.z; + entry.3 += p.r as f32; + entry.4 += p.g as f32; + entry.5 += p.b as f32; + entry.6 = entry.6.max(p.intensity); + entry.7 += 1; + } + } + + let mut fused = PointCloud::new("fused"); + for (_, (sx, sy, sz, sr, sg, sb, mi, n)) in &cells { + let n = *n as f32; + fused.points.push(ColorPoint { + x: sx / n, y: sy / n, z: sz / n, + r: (sr / n) as u8, g: (sg / n) as u8, b: (sb / n) as u8, + intensity: *mi, + }); + } + fused +} + +/// Fetch WiFi occupancy from a remote RuView/brain endpoint. +pub async fn fetch_wifi_occupancy(url: &str) -> anyhow::Result { + let client = reqwest::Client::new(); + let resp: serde_json::Value = client.get(url).send().await?.json().await?; + + let nx = resp.get("nx").and_then(|v| v.as_u64()).unwrap_or(8) as usize; + let ny = resp.get("ny").and_then(|v| v.as_u64()).unwrap_or(8) as usize; + let nz = resp.get("nz").and_then(|v| v.as_u64()).unwrap_or(4) as usize; + + let densities: Vec = resp.get("densities") + .and_then(|v| v.as_array()) + .map(|arr| arr.iter().filter_map(|v| v.as_f64()).collect()) + .unwrap_or_else(|| vec![0.0; nx * ny * nz]); + + let bounds = resp.get("bounds") + .and_then(|v| v.as_array()) + .map(|arr| { + let mut b = [0.0f64; 6]; + for (i, v) in arr.iter().enumerate().take(6) { + b[i] = v.as_f64().unwrap_or(0.0); + } + b + }) + .unwrap_or([0.0, 0.0, 0.0, 5.0, 5.0, 3.0]); + + let occupied_count = densities.iter().filter(|&&d| d > 0.3).count(); + + Ok(OccupancyVolume { densities, nx, ny, nz, bounds, occupied_count }) +} + +/// Generate a demo occupancy volume (room with person). +pub fn demo_occupancy() -> OccupancyVolume { + let nx = 10; + let ny = 10; + let nz = 5; + let mut densities = vec![0.0f64; nx * ny * nz]; + + // Walls (high density at edges) + for iz in 0..nz { + for iy in 0..ny { + for ix in 0..nx { + let idx = iz * ny * nx + iy * nx + ix; + // Edges = walls + if ix == 0 || ix == nx - 1 || iy == 0 || iy == ny - 1 { + densities[idx] = 0.8; + } + // Floor + if iz == 0 { + densities[idx] = 0.6; + } + // Person at center (iz=1-3, ix=4-6, iy=4-6) + if (4..=6).contains(&ix) && (4..=6).contains(&iy) && (1..=3).contains(&iz) { + densities[idx] = 0.9; + } + } + } + } + + let occupied_count = densities.iter().filter(|&&d| d > 0.3).count(); + OccupancyVolume { + densities, nx, ny, nz, + bounds: [0.0, 0.0, 0.0, 5.0, 5.0, 3.0], + occupied_count, + } +} diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/main.rs b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/main.rs new file mode 100644 index 00000000..539f110c --- /dev/null +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/main.rs @@ -0,0 +1,102 @@ +//! ruview-pointcloud — real-time dense point cloud from camera + WiFi CSI +//! +//! Pipeline: Camera → Depth (MiDaS ONNX) → Backproject → Fuse with WiFi occupancy → Stream +//! +//! Usage: +//! ruview-pointcloud serve # start HTTP + WebSocket server +//! ruview-pointcloud capture --frames 1 # capture single frame to PLY +//! ruview-pointcloud demo # generate demo point cloud + +mod depth; +mod pointcloud; +mod fusion; +mod stream; + +use anyhow::Result; +use clap::{Parser, Subcommand}; + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +#[derive(Parser)] +#[command(name = "ruview-pointcloud", version = VERSION)] +struct Cli { + #[command(subcommand)] + command: Commands, +} + +#[derive(Subcommand)] +enum Commands { + /// Start real-time point cloud server (HTTP + WebSocket) + Serve { + #[arg(long, default_value = "0.0.0.0")] + host: String, + #[arg(long, default_value = "9880")] + port: u16, + /// WiFi occupancy source URL (e.g., http://ruvultra:9876) + #[arg(long)] + wifi_source: Option, + }, + /// Capture frames to PLY file + Capture { + #[arg(long, default_value = "1")] + frames: usize, + #[arg(long, default_value = "output.ply")] + output: String, + }, + /// Generate demo point cloud (no camera needed) + Demo, +} + +#[tokio::main] +async fn main() -> Result<()> { + let cli = Cli::parse(); + + match cli.command { + Commands::Serve { host, port, wifi_source } => { + stream::serve(&host, port, wifi_source.as_deref()).await?; + } + Commands::Capture { frames, output } => { + let cloud = depth::capture_depth_cloud(frames).await?; + pointcloud::write_ply(&cloud, &output)?; + println!("Wrote {} points to {output}", cloud.points.len()); + } + Commands::Demo => { + demo().await?; + } + } + + Ok(()) +} + +async fn demo() -> Result<()> { + println!("╔══════════════════════════════════════════════╗"); + println!("║ RuView Dense Point Cloud — Demo ║"); + println!("╚══════════════════════════════════════════════╝"); + println!(); + + // Generate a demo occupancy volume (simulated WiFi tomography) + let occupancy = fusion::demo_occupancy(); + let wifi_cloud = fusion::occupancy_to_pointcloud(&occupancy); + println!("WiFi occupancy: {}x{}x{} voxels → {} points", + occupancy.nx, occupancy.ny, occupancy.nz, wifi_cloud.points.len()); + + // Generate a demo depth cloud (simulated camera) + let depth_cloud = depth::demo_depth_cloud(); + println!("Camera depth: {} points", depth_cloud.points.len()); + + // Fuse + let fused = fusion::fuse_clouds(&[&wifi_cloud, &depth_cloud], 0.05); + println!("Fused: {} points (voxel size=0.05m)", fused.points.len()); + + // Write PLY + pointcloud::write_ply(&fused, "demo_pointcloud.ply")?; + println!("\nWrote: demo_pointcloud.ply"); + + // Write Gaussian splats + let splats = pointcloud::to_gaussian_splats(&fused); + let json = serde_json::to_string_pretty(&splats)?; + std::fs::write("demo_splats.json", &json)?; + println!("Wrote: demo_splats.json ({} splats)", splats.len()); + + Ok(()) +} diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/pointcloud.rs b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/pointcloud.rs new file mode 100644 index 00000000..ab3f9536 --- /dev/null +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/pointcloud.rs @@ -0,0 +1,125 @@ +//! Point cloud types + PLY export + Gaussian splat conversion. + +use serde::{Deserialize, Serialize}; +use std::io::Write; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Point3D { + pub x: f32, + pub y: f32, + pub z: f32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ColorPoint { + pub x: f32, + pub y: f32, + pub z: f32, + pub r: u8, + pub g: u8, + pub b: u8, + pub intensity: f32, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct PointCloud { + pub points: Vec, + pub timestamp_ms: i64, + pub source: String, +} + +impl PointCloud { + pub fn new(source: &str) -> Self { + Self { + points: Vec::new(), + timestamp_ms: chrono::Utc::now().timestamp_millis(), + source: source.to_string(), + } + } + + pub fn add(&mut self, x: f32, y: f32, z: f32, r: u8, g: u8, b: u8, intensity: f32) { + self.points.push(ColorPoint { x, y, z, r, g, b, intensity }); + } + + pub fn bounds(&self) -> ([f32; 3], [f32; 3]) { + if self.points.is_empty() { + return ([0.0; 3], [0.0; 3]); + } + let mut min = [f32::MAX; 3]; + let mut max = [f32::MIN; 3]; + for p in &self.points { + min[0] = min[0].min(p.x); min[1] = min[1].min(p.y); min[2] = min[2].min(p.z); + max[0] = max[0].max(p.x); max[1] = max[1].max(p.y); max[2] = max[2].max(p.z); + } + (min, max) + } +} + +/// Write point cloud to PLY format (ASCII). +pub fn write_ply(cloud: &PointCloud, path: &str) -> anyhow::Result<()> { + let mut f = std::fs::File::create(path)?; + writeln!(f, "ply")?; + writeln!(f, "format ascii 1.0")?; + writeln!(f, "comment Generated by RuView Dense Point Cloud")?; + writeln!(f, "comment Source: {}", cloud.source)?; + writeln!(f, "comment Timestamp: {}", cloud.timestamp_ms)?; + writeln!(f, "element vertex {}", cloud.points.len())?; + writeln!(f, "property float x")?; + writeln!(f, "property float y")?; + writeln!(f, "property float z")?; + writeln!(f, "property uchar red")?; + writeln!(f, "property uchar green")?; + writeln!(f, "property uchar blue")?; + writeln!(f, "property float intensity")?; + writeln!(f, "end_header")?; + for p in &cloud.points { + writeln!(f, "{:.4} {:.4} {:.4} {} {} {} {:.4}", p.x, p.y, p.z, p.r, p.g, p.b, p.intensity)?; + } + Ok(()) +} + +/// Convert point cloud to Gaussian splats for 3D rendering. +#[derive(Serialize, Deserialize)] +pub struct GaussianSplat { + pub center: [f32; 3], + pub color: [f32; 3], + pub opacity: f32, + pub scale: [f32; 3], +} + +pub fn to_gaussian_splats(cloud: &PointCloud) -> Vec { + // Cluster points into voxels and create one Gaussian per cluster + let voxel_size = 0.1; + let mut cells: std::collections::HashMap<(i32, i32, i32), Vec<&ColorPoint>> = std::collections::HashMap::new(); + + for p in &cloud.points { + let key = ( + (p.x / voxel_size).floor() as i32, + (p.y / voxel_size).floor() as i32, + (p.z / voxel_size).floor() as i32, + ); + cells.entry(key).or_default().push(p); + } + + cells.values().map(|pts| { + let n = pts.len() as f32; + let cx = pts.iter().map(|p| p.x).sum::() / n; + let cy = pts.iter().map(|p| p.y).sum::() / n; + let cz = pts.iter().map(|p| p.z).sum::() / n; + let cr = pts.iter().map(|p| p.r as f32).sum::() / n / 255.0; + let cg = pts.iter().map(|p| p.g as f32).sum::() / n / 255.0; + let cb = pts.iter().map(|p| p.b as f32).sum::() / n / 255.0; + + // Scale based on point spread + let sx = pts.iter().map(|p| (p.x - cx).abs()).sum::() / n + 0.01; + let sy = pts.iter().map(|p| (p.y - cy).abs()).sum::() / n + 0.01; + let sz = pts.iter().map(|p| (p.z - cz).abs()).sum::() / n + 0.01; + + GaussianSplat { + center: [cx, cy, cz], + color: [cr, cg, cb], + opacity: (n / 10.0).min(1.0), + scale: [sx, sy, sz], + } + }).collect() +} diff --git a/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/stream.rs b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/stream.rs new file mode 100644 index 00000000..69f8747b --- /dev/null +++ b/rust-port/wifi-densepose-rs/crates/wifi-densepose-pointcloud/src/stream.rs @@ -0,0 +1,186 @@ +//! WebSocket + HTTP server for real-time point cloud streaming. + +use crate::depth; +use crate::fusion; +use crate::pointcloud; +use axum::{ + extract::State, + response::{Html, IntoResponse}, + routing::get, + Json, Router, +}; +use std::sync::Arc; + +struct AppState { + wifi_source: Option, +} + +pub async fn serve(host: &str, port: u16, wifi_source: Option<&str>) -> anyhow::Result<()> { + let state = Arc::new(AppState { + wifi_source: wifi_source.map(|s| s.to_string()), + }); + + let app = Router::new() + .route("/", get(index)) + .route("/api/cloud", get(api_cloud)) + .route("/api/splats", get(api_splats)) + .route("/api/status", get(api_status)) + .route("/health", get(api_health)) + .with_state(state); + + let addr = format!("{host}:{port}"); + println!("╔══════════════════════════════════════════════╗"); + println!("║ RuView Dense Point Cloud Server ║"); + println!("╚══════════════════════════════════════════════╝"); + println!(" HTTP: http://{addr}"); + println!(" WebSocket: ws://{addr}/ws"); + println!(" API: http://{addr}/api/cloud"); + println!(" Viewer: http://{addr}/"); + + let listener = tokio::net::TcpListener::bind(&addr).await?; + axum::serve(listener, app).await?; + Ok(()) +} + +async fn api_cloud() -> Json { + let occupancy = fusion::demo_occupancy(); + let wifi_cloud = fusion::occupancy_to_pointcloud(&occupancy); + let depth_cloud = depth::demo_depth_cloud(); + let fused = fusion::fuse_clouds(&[&wifi_cloud, &depth_cloud], 0.05); + let (min, max) = fused.bounds(); + + Json(serde_json::json!({ + "points": fused.points.len(), + "bounds_min": min, + "bounds_max": max, + "sources": ["camera_depth", "wifi_occupancy"], + "cloud": fused.points.iter().take(1000).collect::>(), + })) +} + +async fn api_splats() -> Json { + let occupancy = fusion::demo_occupancy(); + let wifi_cloud = fusion::occupancy_to_pointcloud(&occupancy); + let depth_cloud = depth::demo_depth_cloud(); + let fused = fusion::fuse_clouds(&[&wifi_cloud, &depth_cloud], 0.05); + let splats = pointcloud::to_gaussian_splats(&fused); + + Json(serde_json::json!({ + "splats": splats, + "count": splats.len(), + "timestamp": chrono::Utc::now().timestamp_millis(), + })) +} + +async fn api_status() -> Json { + Json(serde_json::json!({ + "status": "ok", + "version": env!("CARGO_PKG_VERSION"), + "pipeline": "camera_depth + wifi_occupancy → fused → gaussian_splats", + "fps": 10, + })) +} + +async fn api_health() -> Json { + Json(serde_json::json!({"status": "ok"})) +} + +async fn index() -> Html { + Html(r#" + + + RuView Dense Point Cloud + + + + + +
+

RuView Dense Point Cloud

+
Connecting...
+
+ + +"#.to_string()) +}