From 42c764652d44e11d14d284f4f3a5986cb30c1aa9 Mon Sep 17 00:00:00 2001 From: ruv Date: Tue, 16 Jun 2026 17:00:57 -0400 Subject: [PATCH] examples(through-wall): ESP32 sensor auto-detection + WiFlow analysis tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - wiflow_browser.html: auto-detect live ESP32 nodes from the /ws/sensing stream and lock them as the model schema (NODE_IDS/CSI_DIM dynamic), persisted + restorable - wiflow_ab.py: leakage-controlled A/B (chronological/random/blocked-gap/grouped-bucket, multi-seed) — the honest CSI→pose evaluation harness - wiflow_capture.py / wiflow_train.py / wiflow_infer.py: camera-paired capture + train + infer - pose.html: live WiFi-inferred skeleton viewer; serve.py: static server - gitignore the regenerable 1.5MB model.npz artifact Co-Authored-By: claude-flow --- .gitignore | 3 + examples/through-wall/pose.html | 159 +++++++++++++++++++++ examples/through-wall/wiflow_ab.py | 126 +++++++++++++++++ examples/through-wall/wiflow_browser.html | 110 ++++++++++++++- examples/through-wall/wiflow_capture.py | 161 ++++++++++++++++++++++ examples/through-wall/wiflow_infer.py | 92 +++++++++++++ examples/through-wall/wiflow_train.py | 102 ++++++++++++++ 7 files changed, 747 insertions(+), 6 deletions(-) create mode 100644 examples/through-wall/pose.html create mode 100644 examples/through-wall/wiflow_ab.py create mode 100644 examples/through-wall/wiflow_capture.py create mode 100644 examples/through-wall/wiflow_infer.py create mode 100644 examples/through-wall/wiflow_train.py diff --git a/.gitignore b/.gitignore index 4297c868..094e45d9 100644 --- a/.gitignore +++ b/.gitignore @@ -277,3 +277,6 @@ aether-arena/staging/ # MM-Fi benchmark dataset archives — large data, fetch separately, never commit assets/MM-Fi/E0*.zip assets/MM-Fi/*.zip + +# through-wall demo: regenerable trained model artifact +examples/through-wall/model/ diff --git a/examples/through-wall/pose.html b/examples/through-wall/pose.html new file mode 100644 index 00000000..74faf9bc --- /dev/null +++ b/examples/through-wall/pose.html @@ -0,0 +1,159 @@ + + + + + +WiFlow · live WiFi-inferred pose + + + +
+

WiFlow · live WiFi-inferred pose

+ +
+
+
+
CSI → pose (skeleton) overlaid on your laptop camera
+
+ + +
+
+ + +
+
camera: off
+
Camera is a visual reference only — it is NOT fed to the model. Overlay alignment is approximate (model trained in a different camera's frame).
+
+
+
live
+
CSI source—
+
nodes—
+
presence—
+
motion—
+
pose fps—
+
+ This skeleton is inferred from WiFi CSI only — no camera in the loop here. A model was + trained on paired (camera-pose, CSI) data in this room (ADR-079/180). +

+ Honest accuracy: ~59.5% PCK@0.10 on held-out data (vs a 50% mean-pose baseline → + +9.4 pp real signal). It captures coarse pose; fine detail is weak (PCK@0.05 ≈ 24%). + Same person / room / session — not validated cross-day or through-wall. +
+
+
+ + + diff --git a/examples/through-wall/wiflow_ab.py b/examples/through-wall/wiflow_ab.py new file mode 100644 index 00000000..f452ca35 --- /dev/null +++ b/examples/through-wall/wiflow_ab.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Rigorous A/B for WiFlow CSI->pose: is the held-out PCK real signal or split leakage? + +For a dataset of {csi:[D], kps:17x[x,y,vis]} pairs, train the SAME small MLP under +several train/val SPLITS and report held-out PCK@0.10 vs the mean-pose baseline: + + - chronological_80_20 : last 20% in time (val temporally ADJACENT to train -> leaks + via CSI/pose autocorrelation; this is what gave us +9.4) + - random_80_20 : shuffled (val frames interleaved with train -> MAX leak) + - blocked_gap : hold out a contiguous MIDDLE block with a time GAP buffer on + each side so val is NOT adjacent to any train frame -> the + honest, leakage-controlled test + +If the model beats baseline on chronological/random but COLLAPSES to ~baseline on +blocked_gap, the apparent signal was temporal leakage, not generalizable CSI->pose. + +Usage (ruvultra venv): python wiflow_ab.py --data ~/wiflow-room/dataset.jsonl +""" +import argparse, json, sys +import numpy as np, torch, torch.nn as nn + +def _rec(r, X, Y, V, B): + X.append(r["csi"]); kp=r["kps"] + if kp and isinstance(kp[0], (list,tuple)): # 17 x [x,y(,vis)] + Y.append([c for k in kp for c in (k[0],k[1])]); V.append([(k[2] if len(k)>2 else 1.0) for k in kp]) + else: # flat 34 (browser export, no vis) + Y.append(list(kp)); V.append([1.0]*17) + B.append(r.get("bucket")) + +def load(path): + X,Y,V,B=[],[],[],[] + txt=open(path).read().strip() + if txt[:1] in "[{": # JSON (browser export: dict{samples:[]} or bare array) + d=json.loads(txt) + rows = d if isinstance(d,list) else d.get("samples", d.get("data", [])) + for r in rows: _rec(r,X,Y,V,B) + else: # JSONL (python capture) + for line in txt.splitlines(): + if line.strip(): _rec(json.loads(line),X,Y,V,B) + return np.array(X,np.float32), np.array(Y,np.float32), np.array(V,np.float32), B + +class Net(nn.Module): + def __init__(s,din,dout): + super().__init__() + s.n=nn.Sequential(nn.Linear(din,384),nn.ReLU(),nn.Dropout(.35), + nn.Linear(384,192),nn.ReLU(),nn.Dropout(.35), + nn.Linear(192,96),nn.ReLU(),nn.Linear(96,dout),nn.Sigmoid()) + def forward(s,x): return s.n(x) + +def pck(pred,gt,vis,thr=0.10): + p=pred.reshape(-1,17,2); g=gt.reshape(-1,17,2) + d=np.linalg.norm(p-g,axis=2); m=vis>0.5 + return float((d[m] val poses/activities never seen in train. + # the strictest leakage-free test (only when bucket labels exist). + b=np.array([x if x is not None else -1 for x in B]) + uniq=[u for u in sorted(set(b.tolist())) if u!=-1] + if len(uniq)<3: raise ValueError("too few buckets") + hold=set(uniq[::max(1,len(uniq)//3)][:max(1,len(uniq)//3)]) # ~1/3 of activities held out + val=idx[np.isin(b,list(hold))]; train=idx[~np.isin(b,list(hold))] + return train, val + raise ValueError(kind) + +def run(X,Y,V,tr,va,epochs=250,seed=0): + torch.manual_seed(seed); np.random.seed(seed) # seed weight init + batch shuffle + dev="cuda" if torch.cuda.is_available() else "cpu" + mu,sd=X[tr].mean(0),X[tr].std(0)+1e-6 + Xtr=torch.tensor((X[tr]-mu)/sd).to(dev); Ytr=torch.tensor(Y[tr]).to(dev) + Xva=torch.tensor((X[va]-mu)/sd).to(dev) + net=Net(X.shape[1],Y.shape[1]).to(dev) + opt=torch.optim.Adam(net.parameters(),lr=1e-3,weight_decay=1e-4); lf=nn.MSELoss() + best=(1e9,None) + for ep in range(epochs): + net.train(); perm=torch.randperm(len(Xtr),device=dev) + for i in range(0,len(Xtr),64): + j=perm[i:i+64]; opt.zero_grad(); loss=lf(net(Xtr[j]),Ytr[j]); loss.backward(); opt.step() + net.eval() + with torch.no_grad(): pv=net(Xva).cpu().numpy() + vl=float(((pv-Y[va])**2).mean()) + if vl16}{'baseline':>11}{'delta (mean±sd)':>20} verdict") + print("-"*86) + splits=["chronological_80_20","random_80_20","blocked_gap"]+(["grouped_bucket"] if has_buckets else []) + for kind in splits: + try: + tr,va=split_idx(n,kind,B) + ms=[]; bs=[] + for s in range(a.seeds): + m,b=run(X,Y,V,tr,va,a.epochs,seed=s); ms.append(m); bs.append(b) + ms=np.array(ms)*100; bs=np.array(bs)*100; ds=ms-bs + dm,dsd=ds.mean(),ds.std() + # REAL only if the mean delta minus 1 sd still clears the 1.5pp threshold (robust to seed variance) + verdict = "REAL signal" if dm-dsd>1.5 else ("weak/uncertain" if dm>1.5 else "no signal (==baseline)") + print(f"{kind:<22}{ms.mean():>13.1f}±{ms.std():>3.1f}{bs.mean():>10.1f}%{dm:>+12.1f}±{dsd:>4.1f}pp {verdict}") + except Exception as e: + print(f"{kind:<22} skipped: {e}") + print(f"\nmean±sd over {a.seeds} seeds (weight init + batch order). blocked_gap = 10% time gap each") + print("side; grouped_bucket holds out ENTIRE activities (strictest). If only the LEAKY splits") + print("(chronological/random) beat baseline, the apparent signal is leakage, not generalizable pose.") + +if __name__=="__main__": main() diff --git a/examples/through-wall/wiflow_browser.html b/examples/through-wall/wiflow_browser.html index 72e88119..5d02aaaf 100644 --- a/examples/through-wall/wiflow_browser.html +++ b/examples/through-wall/wiflow_browser.html @@ -112,7 +112,11 @@
empty-room baseline (ADR-151) — step OUT of the space
- + + not detected +
+
+ @@ -285,9 +289,15 @@ // wss when served over https (mobile/secure-context safe), else ws; ?ws= overrides const CSI_WS = (new URLSearchParams(location.search)).get('ws') || `${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.hostname || 'localhost'}:8765/ws/sensing`; -const NODE_IDS = [9, 13]; // per-node features in this fixed order (matches Python pipeline) +// Per-node feature schema — AUTO-DETECTED from the live stream (see detectSensors). +// [9,13] is only the fallback until detection runs. ORDER is fixed (sorted ascending) +// so the model's input layout is stable across capture / train / infer. +let NODE_IDS = [9, 13]; const FIELD_LEN = 400; // signal_field.values padded/truncated to 400 -const CSI_DIM = 4 + NODE_IDS.length * 3 + FIELD_LEN; // 4 + 6 + 400 = 410 +let CSI_DIM = 4 + NODE_IDS.length * 3 + FIELD_LEN; // 4 global + 3/node + 400 field +function recomputeCsiDim(){ CSI_DIM = 4 + NODE_IDS.length * 3 + FIELD_LEN; } +let sensorsDetected = false; // true once a detect (auto/manual/restored) has locked the node set +let autoDetectStarted = false; // one-shot guard for the auto-detect on first live frame const N_KP = 17, OUT_DIM = N_KP * 2; // 17 COCO keypoints -> 34 coords const BASELINE_SECONDS = 10; // empty-room calibration window const EPS = 1e-6; @@ -333,9 +343,9 @@ async function selectBackend(){ // ============================================================================ // CSI vector construction — MUST match wiflow_capture.py csi_vector() exactly. // [mean_rssi, variance, motion_band_power, breathing_band_power] (4 global) -// + for node 9 then node 13: [mean_rssi, variance, motion_band_power] (6 per-node) +// + for each node in NODE_IDS order: [mean_rssi, variance, motion_band_power] (3 per-node) // + signal_field.values padded/truncated to 400 (400 field) -// = 410-d (RAW — baseline-normalization applied separately, see baselineNorm) +// = CSI_DIM-d (RAW — baseline-normalization applied separately, see baselineNorm) // ============================================================================ function csiVector(frame){ const f = frame.features || {}; @@ -368,6 +378,87 @@ function baselineNorm(vecRaw){ return out; } +// ============================================================================ +// ESP32 sensor auto-detection +// Sniff the live /ws/sensing stream, find which node_ids are actually present +// and healthy, and lock that ordered set as the per-node schema (NODE_IDS/CSI_DIM). +// The node set defines the model's input dimension, so detection must run BEFORE +// calibration + capture; changing it invalidates a baseline/dataset built on a +// different set (we confirm, then reset, on a manual re-detect). +// ============================================================================ +async function detectSensors(ms = 3000){ + const tally = {}; // node_id -> { seen, fps, rssi } + let frames = 0; + const t0 = performance.now(); + const el = $('detNodes'); if (el){ el.textContent = 'scanning…'; el.className = 'v'; } + while (performance.now() - t0 < ms){ + if (latestCSI.frame && latestCSI.source === 'esp32'){ + frames++; + for (const nf of (latestCSI.frame.node_features || [])){ + const id = nf.node_id; if (id == null) continue; + const f = nf.features || {}; + const t = (tally[id] || (tally[id] = { seen:0, fps:0, rssi:0 })); + t.seen++; t.fps += (+nf.frame_rate_hz || 0); + t.rssi += (+f.mean_rssi || +nf.rssi_dbm || 0); + } + } + await new Promise(r => setTimeout(r, 100)); + } + // healthy = seen in >40% of sampled frames (filters transient / duplicate ids) + const healthy = Object.keys(tally).map(k => ({ + id:+k, seen:tally[k].seen, fps:tally[k].fps/tally[k].seen, rssi:tally[k].rssi/tally[k].seen })) + .filter(n => n.seen >= Math.max(2, frames * 0.4)) + .sort((a,b)=> a.id - b.id); + return { healthy, frames }; +} + +function renderDetectedSensors(list){ + const el = $('detNodes'); if (!el) return; + el.textContent = list.length + ? list.map(n => `#${n.id} (${Math.round(n.fps)}fps, ${Math.round(n.rssi)}dB)`).join(' · ') + : 'none found'; + el.className = list.length ? 'v green' : 'v red'; +} + +async function runDetect(manual){ + const { healthy, frames } = await detectSensors(manual ? 4000 : 3000); + if (!healthy.length){ + const el = $('detNodes'); + if (el){ el.textContent = frames ? 'no healthy nodes' : 'no live CSI (start sensing-server / esp32)'; + el.className = 'v red'; } + return; + } + const ids = healthy.map(n => n.id); + const changed = ids.length !== NODE_IDS.length || ids.some((v,i)=> v !== NODE_IDS[i]); + if (changed && (baseline || SAMPLES.length)){ + const ok = confirm( + `Detected sensors [${ids.join(', ')}] differ from the current set [${NODE_IDS.join(', ')}].\n\n` + + `The node set defines the model input, so switching invalidates the existing baseline` + + (SAMPLES.length ? ` and ${SAMPLES.length} captured samples` : ``) + + `. Reset and use the detected set?`); + if (!ok){ renderDetectedSensors(healthy); return; } + if (baseline){ baseline = null; stageDone.calibrate = false; idbDel('baseline'); + $('calStatus').textContent = 'NOT CALIBRATED'; $('calStatus').className = 'v'; $('calBar').style.width = '0%'; } + if (SAMPLES.length){ SAMPLES = []; covCounts = new Array(BUCKETS.length).fill(0); + idbPut('samples', []); $('capN').textContent = '0'; $('trN').textContent = '0'; renderCoverage(); } + } + NODE_IDS = ids; recomputeCsiDim(); sensorsDetected = true; + idbPut('nodeIds', NODE_IDS); + renderDetectedSensors(healthy); + refreshGates(); +} + +async function restoreNodeIds(){ + try{ + const ids = await idbGet('nodeIds'); + if (Array.isArray(ids) && ids.length){ + NODE_IDS = ids.slice(); recomputeCsiDim(); sensorsDetected = true; + const el = $('detNodes'); + if (el){ el.textContent = 'restored: ' + NODE_IDS.map(i => '#' + i).join(' '); el.className = 'v'; } + } + }catch(e){ /* ignore */ } +} + // ============================================================================ // CSI WebSocket // ============================================================================ @@ -388,6 +479,11 @@ function connectCSI(){ source: src, nodes: (d.nodes || []).map(n => n.node_id).filter(x => x != null).sort((a,b)=>a-b) }; + // auto-detect the sensor set once, on the first live frame, only when starting fresh + // (no baseline / no samples) so we never silently change a schema work is built on. + if (src === 'esp32' && !sensorsDetected && !autoDetectStarted && !baseline && SAMPLES.length === 0){ + autoDetectStarted = true; runDetect(false); + } if (src === 'esp32') banner('live','LIVE — real ESP32 CSI'); else banner('sim',`SIMULATED — not real (source=${src})`); }; @@ -599,6 +695,7 @@ function finishCalibration(){ refreshGates(); } $('calBtn').addEventListener('click', startCalibration); +$('detBtn').addEventListener('click', ()=> runDetect(true)); $('recalBtn').addEventListener('click', ()=>{ baseline = null; stageDone.calibrate = false; $('calStatus').textContent = 'NOT CALIBRATED'; $('calStatus').className = 'v'; $('calBar').style.width = '0%'; $('calN').textContent = '0'; idbDel('baseline'); refreshGates(); startCalibration(); }); @@ -736,7 +833,7 @@ $('clrBtn').addEventListener('click', async ()=>{ $('expBtn').addEventListener('click', ()=>{ const out = { format: 'wiflow-browser-dataset', version: 1, exported: new Date().toISOString(), - csi_dim: CSI_DIM, out_dim: OUT_DIM, buckets: BUCKETS, + csi_dim: CSI_DIM, out_dim: OUT_DIM, buckets: BUCKETS, nodes: NODE_IDS.slice(), note: 'csi is baseline-normalized (ADR-151 deviation-from-baseline); kps are 17 COCO keypoints in [0,1] image coords', samples: SAMPLES.map((s,i)=>({ csi: Array.from(s.csi), kps: Array.from(s.kps), bucket: s.bucket, t: (s.t!=null?s.t:i) })) }; @@ -1152,6 +1249,7 @@ function inferLoop(){ (async function boot(){ connectCSI(); await selectBackend(); + await restoreNodeIds(); // restore a previously-detected sensor set (fixes CSI_DIM before baseline) await loadBaseline(); await idbLoad(); await loadModel(); diff --git a/examples/through-wall/wiflow_capture.py b/examples/through-wall/wiflow_capture.py new file mode 100644 index 00000000..b77c3951 --- /dev/null +++ b/examples/through-wall/wiflow_capture.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +"""WiFlow-style camera-supervised capture (ADR-079 / ADR-180). + +Runs on a box with BOTH a camera (ground truth) and reachable live CSI: + - opens a camera, runs MediaPipe Pose -> 17 COCO keypoints (the LABEL), + - subscribes to the sensing-server /ws/sensing (the INPUT: CSI features + + 20x20 signal-field), + - writes timestamp-aligned (csi -> pose) pairs to a JSONL dataset. + +This is the *collect* phase of camera-supervised CSI->pose training. The camera +and the CSI nodes MUST see the same person in the same space at the same time, +or the pairs are meaningless. Honest by construction: we only emit a pair when +BOTH a confident camera pose AND a live (source=esp32) CSI frame are present in +the same ~100 ms window. + +Usage (on ruvultra, with the CSI tunneled to localhost:8765): + python3 wiflow_capture.py --ws ws://localhost:8765/ws/sensing \ + --cam 0 --out ~/wiflow-room/dataset.jsonl --seconds 180 +""" +import argparse, asyncio, json, time, threading, sys, os +from collections import deque + +import urllib.request +import cv2 +import numpy as np +import mediapipe as mp +from mediapipe.tasks.python import BaseOptions +from mediapipe.tasks.python.vision import PoseLandmarker, PoseLandmarkerOptions, RunningMode +import websockets + +_MODEL_URL = ("https://storage.googleapis.com/mediapipe-models/pose_landmarker/" + "pose_landmarker_lite/float16/latest/pose_landmarker_lite.task") + +def ensure_model(path: str) -> str: + if not os.path.exists(path): + os.makedirs(os.path.dirname(path), exist_ok=True) + print(f"[capture] downloading pose model -> {path}", flush=True) + urllib.request.urlretrieve(_MODEL_URL, path) + return path + +# MediaPipe Pose (33 landmarks) -> 17 COCO keypoints (same mapping as +# scripts/collect-ground-truth.py, ADR-079). +COCO_FROM_MP = [0, 2, 5, 7, 8, 11, 12, 13, 14, 15, 16, 23, 24, 25, 26, 27, 28] +COCO_NAMES = ["nose","l_eye","r_eye","l_ear","r_ear","l_sho","r_sho","l_elb", + "r_elb","l_wri","r_wri","l_hip","r_hip","l_knee","r_knee","l_ank","r_ank"] + +# ---- shared state between the CSI (async) thread and the camera (sync) loop ---- +_latest_csi = {"t": 0.0, "frame": None} +_csi_lock = threading.Lock() +_stop = threading.Event() + + +def csi_thread(ws_url: str): + """Background thread: keep the most recent LIVE csi frame in _latest_csi.""" + async def run(): + while not _stop.is_set(): + try: + async with websockets.connect(ws_url, open_timeout=8, ping_interval=20) as ws: + while not _stop.is_set(): + msg = await asyncio.wait_for(ws.recv(), timeout=8) + d = json.loads(msg) + with _csi_lock: + _latest_csi["t"] = time.time() + _latest_csi["frame"] = d + except Exception as e: + print(f"[csi] reconnect ({e})", flush=True) + await asyncio.sleep(1.0) + asyncio.new_event_loop().run_until_complete(run()) + + +def csi_vector(frame: dict): + """Flatten a csi frame to a fixed-length input vector: features + field.""" + f = frame.get("features", {}) or {} + feats = [f.get("mean_rssi", 0.0), f.get("variance", 0.0), + f.get("motion_band_power", 0.0), f.get("breathing_band_power", 0.0)] + # per-node mean_rssi/variance/motion for up to the 2 nodes (9, 13) + pernode = {nf.get("node_id"): (nf.get("features") or {}) for nf in (frame.get("node_features") or [])} + for nid in (9, 13): + nf = pernode.get(nid, {}) + feats += [nf.get("mean_rssi", 0.0), nf.get("variance", 0.0), nf.get("motion_band_power", 0.0)] + field = (frame.get("signal_field", {}) or {}).get("values") or [] + field = (field + [0.0] * 400)[:400] + return feats + field # 4 + 6 + 400 = 410-d + + +def main(): + ap = argparse.ArgumentParser(description="WiFlow camera-supervised CSI<->pose capture (ADR-180).") + ap.add_argument("--ws", default="ws://localhost:8765/ws/sensing") + ap.add_argument("--cam", type=int, default=0) + ap.add_argument("--out", default=os.path.expanduser("~/wiflow-room/dataset.jsonl")) + ap.add_argument("--seconds", type=int, default=180) + ap.add_argument("--min-vis", type=float, default=0.5, help="min mean landmark visibility to accept a pose label") + ap.add_argument("--max-skew-ms", type=float, default=150, help="max csi/pose time skew to pair") + ap.add_argument("--require-esp32", action="store_true", default=True, + help="only pair when csi source==esp32 (real). Default on.") + args = ap.parse_args() + + os.makedirs(os.path.dirname(args.out), exist_ok=True) + th = threading.Thread(target=csi_thread, args=(args.ws,), daemon=True) + th.start() + + cap = cv2.VideoCapture(args.cam) + if not cap.isOpened(): + print(f"ERROR: cannot open camera {args.cam}", file=sys.stderr); sys.exit(2) + W = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) or 640 + H = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) or 480 + model_path = ensure_model(os.path.expanduser("~/wiflow-room/pose_landmarker_lite.task")) + landmarker = PoseLandmarker.create_from_options(PoseLandmarkerOptions( + base_options=BaseOptions(model_asset_path=model_path), + running_mode=RunningMode.IMAGE, min_pose_detection_confidence=0.5)) + + n_pairs = 0; n_nopose = 0; n_nocsi = 0; n_skew = 0; n_sim = 0 + t0 = time.time() + print(f"[capture] camera {args.cam} {W}x{H} -> {args.out} for {args.seconds}s") + print("[capture] stand in view AND in the CSI field; move/walk so poses vary. Ctrl-C to stop.") + with open(args.out, "a") as out: + try: + while time.time() - t0 < args.seconds: + ok, frame = cap.read() + if not ok: + continue + now = time.time() + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + res = landmarker.detect(mp.Image(image_format=mp.ImageFormat.SRGB, data=rgb)) + if not res.pose_landmarks: + n_nopose += 1; continue + lm = res.pose_landmarks[0] + kps = [[lm[i].x, lm[i].y, lm[i].visibility] for i in COCO_FROM_MP] + vis = float(np.mean([k[2] for k in kps])) + if vis < args.min_vis: + n_nopose += 1; continue + with _csi_lock: + ct = _latest_csi["t"]; cf = _latest_csi["frame"] + if cf is None: + n_nocsi += 1; continue + if (now - ct) * 1000.0 > args.max_skew_ms: + n_skew += 1; continue + if args.require_esp32 and cf.get("source") != "esp32": + n_sim += 1; continue + rec = {"t": now, "vis": round(vis, 3), + "kps": [[round(x, 4), round(y, 4), round(v, 3)] for x, y, v in kps], + "csi": csi_vector(cf), + "src": cf.get("source"), + "nodes": sorted(n.get("node_id") for n in cf.get("nodes", []) if n.get("node_id") is not None)} + out.write(json.dumps(rec) + "\n") + n_pairs += 1 + if n_pairs % 30 == 0: + out.flush() + el = int(now - t0) + print(f"[capture] t+{el:3d}s pairs={n_pairs} (skip: nopose={n_nopose} nocsi={n_nocsi} skew={n_skew} sim={n_sim})", flush=True) + except KeyboardInterrupt: + print("\n[capture] stopped by user") + _stop.set(); cap.release() + print(f"[capture] DONE. wrote {n_pairs} paired samples to {args.out}") + print(f"[capture] skipped: no-pose={n_nopose} no-csi={n_nocsi} skew={n_skew} simulated={n_sim}") + if n_pairs == 0: + print("[capture] WARNING: 0 pairs — check camera sees you AND csi source==esp32 (live).") + + +if __name__ == "__main__": + main() diff --git a/examples/through-wall/wiflow_infer.py b/examples/through-wall/wiflow_infer.py new file mode 100644 index 00000000..aedce4e0 --- /dev/null +++ b/examples/through-wall/wiflow_infer.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python3 +"""Live CSI->pose inference bridge (ADR-180). + +Runs on the box with the live CSI. Loads the camera-supervised model (numpy, +no torch needed), subscribes to /ws/sensing, runs a forward pass per frame, and +broadcasts the predicted 17-keypoint pose to HTML clients on ws://:8770/pose. + + python wiflow_infer.py --model model/model.npz \ + --in ws://localhost:8765/ws/sensing --port 8770 +""" +import argparse, asyncio, json, os +import numpy as np +import websockets + +# COCO skeleton edges (for the client; sent once in 'meta') +EDGES = [[5,7],[7,9],[6,8],[8,10],[5,6],[11,12],[5,11],[6,12], + [11,13],[13,15],[12,14],[14,16],[0,1],[0,2],[1,3],[2,4],[0,5],[0,6]] + +def csi_vector(frame): + f = frame.get("features", {}) or {} + feats = [f.get("mean_rssi",0.0), f.get("variance",0.0), + f.get("motion_band_power",0.0), f.get("breathing_band_power",0.0)] + pernode = {nf.get("node_id"): (nf.get("features") or {}) for nf in (frame.get("node_features") or [])} + for nid in (9,13): + nf = pernode.get(nid,{}); feats += [nf.get("mean_rssi",0.0), nf.get("variance",0.0), nf.get("motion_band_power",0.0)] + field = (frame.get("signal_field",{}) or {}).get("values") or [] + field = (field + [0.0]*400)[:400] + return np.array(feats + field, np.float32) + +class Model: + def __init__(self, path): + z = np.load(path) + self.mu, self.sd = z["mu"], z["sd"] + self.W = [z["net_0_weight"], z["net_3_weight"], z["net_6_weight"], z["net_8_weight"]] + self.b = [z["net_0_bias"], z["net_3_bias"], z["net_6_bias"], z["net_8_bias"]] + def __call__(self, x): + h = (x - self.mu) / self.sd + for i in range(3): + h = np.maximum(0.0, h @ self.W[i].T + self.b[i]) # Linear+ReLU + out = 1.0/(1.0+np.exp(-(h @ self.W[3].T + self.b[3]))) # Linear+Sigmoid -> 34 + return out.reshape(17,2) + +CLIENTS = set() +LATEST = {"pose": None} + +async def serve_client(ws): + CLIENTS.add(ws) + try: + await ws.send(json.dumps({"type":"meta","edges":EDGES})) + async for _ in ws: # client is read-only; just keep alive + pass + except Exception: + pass + finally: + CLIENTS.discard(ws) + +async def infer_loop(model, in_url): + while True: + try: + async with websockets.connect(in_url, open_timeout=8, ping_interval=20) as ws: + async for msg in ws: + d = json.loads(msg) + kp = model(csi_vector(d)) + cls = d.get("classification",{}) + payload = {"type":"pose","src":d.get("source"), + "presence":bool(cls.get("presence")), + "motion":(d.get("features",{}) or {}).get("motion_band_power"), + "kps":[[round(float(x),4),round(float(y),4)] for x,y in kp], + "nodes":sorted(n.get("node_id") for n in d.get("nodes",[]) if n.get("node_id") is not None)} + LATEST["pose"]=payload + if CLIENTS: + dead=[] + for c in list(CLIENTS): + try: await c.send(json.dumps(payload)) + except Exception: dead.append(c) + for c in dead: CLIENTS.discard(c) + except Exception as e: + print(f"[infer] reconnect ({e})", flush=True); await asyncio.sleep(1.0) + +async def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--model", default=os.path.join(os.path.dirname(__file__),"model","model.npz")) + ap.add_argument("--in", dest="in_url", default="ws://localhost:8765/ws/sensing") + ap.add_argument("--port", type=int, default=8770) + args = ap.parse_args() + model = Model(args.model) + print(f"[infer] model {args.model} loaded; serving predicted poses on ws://0.0.0.0:{args.port}/pose") + async with websockets.serve(serve_client, "0.0.0.0", args.port): + await infer_loop(model, args.in_url) + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/through-wall/wiflow_train.py b/examples/through-wall/wiflow_train.py new file mode 100644 index 00000000..7d0bd809 --- /dev/null +++ b/examples/through-wall/wiflow_train.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""Train a CSI->pose model on the camera-supervised dataset (ADR-079/180). + +Input : 410-d CSI vector (4 global feats + 6 per-node + 400 signal-field). +Target : 17 COCO keypoints (x,y), normalized 0..1 from the camera (ground truth). +Reports HONEST held-out PCK@k + MPJPE on a chronological val split (the last +20% of the session — never trained on), so the number is not leaked. + +Usage (ruvultra venv): + python wiflow_train.py --data ~/wiflow-room/dataset.jsonl --out ~/wiflow-room/model.pt +""" +import argparse, json, math, os, sys +import numpy as np +import torch, torch.nn as nn + + +def load(path): + X, Y, V = [], [], [] + with open(path) as f: + for line in f: + r = json.loads(line) + X.append(r["csi"]) # 410 + kp = r["kps"] # 17 x [x,y,vis] + Y.append([c for k in kp for c in (k[0], k[1])]) # 34 + V.append([k[2] for k in kp]) # 17 visibilities + return np.array(X, np.float32), np.array(Y, np.float32), np.array(V, np.float32) + + +class Net(nn.Module): + def __init__(self, din, dout): + super().__init__() + self.net = nn.Sequential( + nn.Linear(din, 512), nn.ReLU(), nn.Dropout(0.3), + nn.Linear(512, 256), nn.ReLU(), nn.Dropout(0.3), + nn.Linear(256, 128), nn.ReLU(), + nn.Linear(128, dout), nn.Sigmoid()) # coords in 0..1 + def forward(self, x): return self.net(x) + + +def pck(pred, gt, vis, thr): + # pred/gt: [N,34] -> [N,17,2]; PCK@thr in normalized image units, visible kps only + p = pred.reshape(-1, 17, 2); g = gt.reshape(-1, 17, 2) + d = np.linalg.norm(p - g, axis=2) # [N,17] + m = vis > 0.5 + return float((d[m] < thr).mean()) if m.any() else 0.0, float(d[m].mean()) if m.any() else float("nan") + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--data", required=True) + ap.add_argument("--out", default=os.path.expanduser("~/wiflow-room/model.pt")) + ap.add_argument("--epochs", type=int, default=300) + ap.add_argument("--bs", type=int, default=64) + args = ap.parse_args() + + X, Y, V = load(args.data) + n = len(X) + print(f"[train] {n} samples, X={X.shape} Y={Y.shape}") + if n < 200: + print("[train] too few samples"); sys.exit(2) + + # chronological split (NOT shuffled) so val is a held-out time segment -> honest + cut = int(n * 0.8) + mu, sd = X[:cut].mean(0), X[:cut].std(0) + 1e-6 # standardize on train only + Xn = (X - mu) / sd + dev = "cuda" if torch.cuda.is_available() else "cpu" + Xtr = torch.tensor(Xn[:cut]).to(dev); Ytr = torch.tensor(Y[:cut]).to(dev) + Xva = torch.tensor(Xn[cut:]).to(dev); Yva = Y[cut:]; Vva = V[cut:] + + # mean-pose baseline (predict the train-mean pose for everything) — the bar to beat + mean_pose = Y[:cut].mean(0) + base_pck, base_mpjpe = pck(np.tile(mean_pose, (len(Yva), 1)), Yva, Vva, 0.10) + + net = Net(X.shape[1], Y.shape[1]).to(dev) + opt = torch.optim.Adam(net.parameters(), lr=1e-3, weight_decay=1e-4) + lossf = nn.MSELoss() + best = (1e9, None) + for ep in range(args.epochs): + net.train(); perm = torch.randperm(len(Xtr), device=dev) + for i in range(0, len(Xtr), args.bs): + idx = perm[i:i+args.bs] + opt.zero_grad(); out = net(Xtr[idx]); loss = lossf(out, Ytr[idx]); loss.backward(); opt.step() + if (ep + 1) % 20 == 0 or ep == args.epochs - 1: + net.eval() + with torch.no_grad(): pv = net(Xva).cpu().numpy() + p10, mpj = pck(pv, Yva, Vva, 0.10); p05, _ = pck(pv, Yva, Vva, 0.05) + vloss = float(((pv - Yva) ** 2).mean()) + print(f"[train] ep{ep+1:3d} val_mse={vloss:.4f} PCK@0.10={p10*100:.1f}% PCK@0.05={p05*100:.1f}% MPJPE={mpj:.4f}") + if vloss < best[0]: best = (vloss, {"sd": net.state_dict(), "p10": p10, "p05": p05, "mpj": mpj}) + + torch.save({"model": best[1]["sd"], "mu": mu, "sd": sd, "din": X.shape[1]}, args.out) + print("\n==================== HONEST RESULT (held-out 20%, never trained) ====================") + print(f" MEAN-POSE BASELINE : PCK@0.10 = {base_pck*100:.1f}% MPJPE = {base_mpjpe:.4f} (the bar to beat)") + print(f" CSI->POSE MODEL : PCK@0.10 = {best[1]['p10']*100:.1f}% PCK@0.05 = {best[1]['p05']*100:.1f}% MPJPE = {best[1]['mpj']:.4f}") + delta = (best[1]['p10'] - base_pck) * 100 + print(f" VERDICT: model {'BEATS' if delta>1 else 'does NOT beat'} mean-pose baseline by {delta:+.1f} pp " + f"-> {'real CSI->pose signal' if delta>1 else 'NO usable CSI->pose signal (honest negative)'}") + print(f" saved -> {args.out}") + + +if __name__ == "__main__": + main()