diff --git a/examples/three.js/.gitignore b/examples/three.js/.gitignore new file mode 100644 index 00000000..b59f7f6a --- /dev/null +++ b/examples/three.js/.gitignore @@ -0,0 +1,10 @@ +# Mixamo FBX downloads — too large + license boundary. Get your own from +# mixamo.com (FBX Binary + T-Pose / Without Skin), drop into assets/. +*.fbx + +# Diagnostic / debug screenshots from a dev session. Official screenshots +# live in screenshots/ and are committed; these underscore-prefixed ones +# are scratch. +_diag-*.png +_demo-mode-shot*.png +_PROOF-*.png diff --git a/examples/three.js/README.md b/examples/three.js/README.md new file mode 100644 index 00000000..893d51da --- /dev/null +++ b/examples/three.js/README.md @@ -0,0 +1,77 @@ +# three.js demos + +Five progressively richer browser demos of the ADR-097 sensing-helpers scene, +ending with a live MediaPipe-Pose → Mixamo X Bot retargeting pipeline driven +by a real ESP32 CSI feed. + +## Run them + +```bash +python examples/three.js/server/serve-demo.py +# then open one of the URLs the script prints +``` + +`server/serve-demo.py` is a tiny `ThreadingHTTPServer` with aggressive +no-cache headers — the stdlib `http.server` is single-threaded and times out +on the parallel script + FBX fetches the demos make. + +## Demos + +| # | File | What it shows | +|---|------|---------------| +| 01 | [`demos/01-helpers.html`](demos/01-helpers.html) | Plain ADR-097 helpers in the point-cloud viewer | +| 02 | [`demos/02-cinematic.html`](demos/02-cinematic.html) | Cinematic camera + pseudo-CSI visualization on top of #01 | +| 03 | [`demos/03-skinned.html`](demos/03-skinned.html) | GLTF skinned mesh + additive animation blending | +| 04 | [`demos/04-skinned-fbx.html`](demos/04-skinned-fbx.html) | Mixamo X Bot loaded from FBX in the ADR-097 scene | +| 05 | [`demos/05-skinned-realtime.html`](demos/05-skinned-realtime.html) | Webcam → MediaPipe Pose Heavy → Mixamo IK retarget, live ESP32 CSI overlay | + +| Screenshot | | +|---|---| +| ![01](screenshots/01-helpers.png) | ![02](screenshots/02-cinematic.png) | +| ![03](screenshots/03-skinned.png) | ![04](screenshots/04-skinned-fbx.png) | +| ![05](screenshots/05-skinned-realtime.png) | | + +## Layout + +``` +examples/three.js/ +├── README.md +├── .gitignore +├── demos/ # 5 self-contained HTML demos +│ ├── 01-helpers.html +│ ├── 02-cinematic.html +│ ├── 03-skinned.html +│ ├── 04-skinned-fbx.html +│ └── 05-skinned-realtime.html +├── screenshots/ # one PNG per demo +│ └── 0N-*.png +├── server/ +│ ├── serve-demo.py # local HTTP server with no-cache headers +│ └── ruvultra-csi-bridge.py # ESP32 CSI WebSocket bridge (ruvultra:8766) +└── assets/ + └── X Bot.fbx # gitignored — get your own from mixamo.com + # (FBX Binary, T-Pose, Without Skin) + # used by demos 04 and 05 +``` + +## Mixamo X Bot + +Demos 04 and 05 expect `assets/X Bot.fbx`. It's gitignored (size + license +boundary). Download yours from [mixamo.com](https://mixamo.com): pick the +"X Bot" character, export as **FBX Binary**, **T-Pose**, **Without Skin**, +and drop it into `assets/`. + +## Live ESP32 CSI overlay (demo 05 only) + +`server/ruvultra-csi-bridge.py` is the systemd-deployable bridge that runs on +the `ruvultra` host (over Tailscale). It listens for ESP32-S3 CSI on UDP and +re-broadcasts it as WebSocket frames at `ws://ruvultra:8766/csi`. Demo 05 +auto-connects; if the socket is down, it falls back to the bundled idle clip +plus a synthetic CSI driver. + +## Open issues + +- [#583](https://github.com/ruvnet/RuView/issues/583) — head/face tracking + fidelity in `05-skinned-realtime.html`. Recommended fix: swap MediaPipe + Pose Heavy for MediaPipe Holistic (same API, adds 468-point face mesh + + hand landmarks for proper PnP head pose and finger curl tracking). diff --git a/examples/three.js/demos/01-helpers.html b/examples/three.js/demos/01-helpers.html new file mode 100644 index 00000000..e915fd13 --- /dev/null +++ b/examples/three.js/demos/01-helpers.html @@ -0,0 +1,587 @@ + + + + + + RuView · ADR-097 · three.js helpers in the point cloud viewer + + + + + + +
+

RuView · Helpers Demo

+
ADR-097 · three.js helpers for the point cloud viewer
+
Scene● SYNTHETIC
+
Skeleton17 kpts · COCO
+
Point cloud— pts
+
Sensor nodes4 · multistatic
+
Frame rate— fps
+
Bbox volume— m³
+
+ +
+

Helpers

+ + + + + +
+ +
+

Scene

+
COCO-17 keypoints (yellow)
+
Bones (white lines)
+
Face point cloud (cyan→white)
+
ESP32 sensor nodes
+
+ +
+ ADR-097 · three.js helpers +
+ + + + diff --git a/examples/three.js/demos/02-cinematic.html b/examples/three.js/demos/02-cinematic.html new file mode 100644 index 00000000..0ec1ff82 --- /dev/null +++ b/examples/three.js/demos/02-cinematic.html @@ -0,0 +1,1034 @@ + + + + + + RuView · Cinematic · ADR-097 helpers + pseudo-CSI visualization + + + + + + + + + + + + +
+
+
+ +
+

RuView · Cinematic

+
ADR-097 · pseudo-CSI visualization layer
+
Subject● Tracked
+
Posewalking
+
Heart rate— bpm
+
Breathing— bpm
+
Mesh nodes4 · multistatic
+
Coherence— %
+
Tomographyidle
+
Bbox vol— m³
+
Render— fps
+
+ +
+

Helpers · ADR-097

+ + + + + + + +
+ +
+

Per-node CSI · synthetic

+
N1·BL
—
+
N2·BR
—
+
N3·FL
—
+
N4·FR
—
+
CSI amplitude derived from distance-to-keypoint + Doppler + thermal noise. Drives bone glow, ping coherence, tomography trigger threshold.
+
+ +
+ RuView · Seldon Vault +
multistatic wifi pose · ADR-097
+
+ +
+ ADR-097 · three.js helpers · cinematic +
+ + + + diff --git a/examples/three.js/demos/03-skinned.html b/examples/three.js/demos/03-skinned.html new file mode 100644 index 00000000..2d2ea73b --- /dev/null +++ b/examples/three.js/demos/03-skinned.html @@ -0,0 +1,854 @@ + + + + + + RuView · Skinned · ADR-097 + GLTF skinned mesh + additive animation blending + + + + + + + + + + + + + +
+
+
+ +
▸ Loading skinned subject · Xbot.glb · 2.9 MB
+ +
+

RuView · Skinned

+
ADR-097 · GLTF skinned mesh · additive animation blending
+
Subject● Tracked
+
ModelXbot.glb · 14k tris
+
Base animwalk
+
AdditiveheadShake · 0.40
+
Mesh nodes4 · multistatic
+
Coherence— %
+
Heart rate— bpm
+
Bbox vol— m³
+
Render— fps
+
+ +
+

AnimationMixer

+
+
Base · loops
+ + + +
+
+
Additive · layered
+ + + + +
+
+
+ add weight + + 0.40 +
+
+ time scale + + 1.00 +
+
+
+ +
+

Per-node CSI

+
N1·BL
—
+
N2·BR
—
+
N3·FL
—
+
N4·FR
—
+
+ +
+

ADR-097 helpers

+ + + + + + + +
+ +
+ RuView · Seldon Vault +
skinned · ADR-097 · CCDIKSolver next
+
+ +
+ additive blend + skinning IK +
+ + + + diff --git a/examples/three.js/demos/04-skinned-fbx.html b/examples/three.js/demos/04-skinned-fbx.html new file mode 100644 index 00000000..8353c015 --- /dev/null +++ b/examples/three.js/demos/04-skinned-fbx.html @@ -0,0 +1,961 @@ + + + + + + RuView · Skinned (FBX) · Mixamo X Bot in the ADR-097 helpers scene + + + + + + + + + + + + + + + +
+
+
+
▸ Loading skinned subject · X Bot.fbx
+ +
+

RuView · Skinned (FBX)

+
ADR-097 · Mixamo X Bot · loaded via FBXLoader
+
Subject● Tracked
+
SourceX Bot.fbx
+
FormatFBX 7700 · 1.75 MB
+
Bones—
+
Animation—
+
Mesh nodes4 · multistatic
+
Coherence— %
+
Heart rate— bpm
+
Bbox vol— m³
+
Render— fps
+
+ +
+

AnimationMixer

+
+ clips + +
+
+ time scale + + 1.00 +
+ +
+ +
+

Per-node CSI

+
N1·BL
—
+
N2·BR
—
+
N3·FL
—
+
N4·FR
—
+
+ +
+

ADR-097 helpers

+ + + + + + + + +
+ +
+ RuView · Seldon Vault +
FBXLoader · Mixamo · ADR-097
+
+ + + + diff --git a/examples/three.js/demos/05-skinned-realtime.html b/examples/three.js/demos/05-skinned-realtime.html new file mode 100644 index 00000000..a366096b --- /dev/null +++ b/examples/three.js/demos/05-skinned-realtime.html @@ -0,0 +1,2141 @@ + + + + + + + + + + RuView · Skinned Realtime · MediaPipe Pose → Mixamo IK retargeting + + + + + + + + + + + + + + + + + +
+
+
+
▸ Loading skinned subject · X Bot.fbx
+ +
+

RuView · Skinned Realtime

+
MediaPipe Pose → Mixamo direct-retargeting · live CSI from real keypoints
+
Subject● Idle (no webcam)
+
SourceX Bot.fbx · 1.75 MB
+
Bones—
+
Pose trackeridle
+
Tracking conf— %
+
Retargets0 / 12
+
RSSI / Wrist L— m
+
Yield / Wrist R— m
+
Bbox vol— m³
+
Render— fps
+
+ +
+

MediaPipe Pose

+
Webcam disabled
+
+ + +
+
+
Landmarks0 / 33
+
Visible0 / 33
+
Pose fps— fps
+
+ + + +
build 2026-05-15-fps-tune · default Holistic@Full 20fps · ?cnn=2 ?infer=30 to crank
+
+ +
+

Per-node CSI · LIVE

+
N1·BL
—
+
N2·BR
—
+
N3·FL
—
+
N4·FR
—
+
connecting to ESP32-S3 via ruvultra (Tailscale ws://100.104.125.72:8766)…
+
+ +
+

ADR-097 helpers

+ + + + + + + + +
+ +
+ RuView · Seldon Vault +
Live · MediaPipe Pose · Mixamo retarget
+
+ + + + diff --git a/examples/three.js/screenshots/01-helpers.png b/examples/three.js/screenshots/01-helpers.png new file mode 100644 index 00000000..4755e996 Binary files /dev/null and b/examples/three.js/screenshots/01-helpers.png differ diff --git a/examples/three.js/screenshots/02-cinematic.png b/examples/three.js/screenshots/02-cinematic.png new file mode 100644 index 00000000..c4737196 Binary files /dev/null and b/examples/three.js/screenshots/02-cinematic.png differ diff --git a/examples/three.js/screenshots/03-skinned.png b/examples/three.js/screenshots/03-skinned.png new file mode 100644 index 00000000..713aa593 Binary files /dev/null and b/examples/three.js/screenshots/03-skinned.png differ diff --git a/examples/three.js/screenshots/04-skinned-fbx.png b/examples/three.js/screenshots/04-skinned-fbx.png new file mode 100644 index 00000000..b747f6dc Binary files /dev/null and b/examples/three.js/screenshots/04-skinned-fbx.png differ diff --git a/examples/three.js/screenshots/05-skinned-realtime.png b/examples/three.js/screenshots/05-skinned-realtime.png new file mode 100644 index 00000000..58268eca Binary files /dev/null and b/examples/three.js/screenshots/05-skinned-realtime.png differ diff --git a/examples/three.js/server/ruvultra-csi-bridge.py b/examples/three.js/server/ruvultra-csi-bridge.py new file mode 100644 index 00000000..1bd4fd38 --- /dev/null +++ b/examples/three.js/server/ruvultra-csi-bridge.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +"""ruvultra → browser CSI bridge. + +Reads adaptive_ctrl tick lines from the ESP32-S3 RuView firmware on +/dev/ttyACM0 and forwards normalized per-node metrics over a WebSocket +that the helpers-skinned-realtime demo can subscribe to via Tailscale. + +Sample serial line (1 Hz cadence from firmware): + I (22890561) adaptive_ctrl: medium tick: state=6 yield=15pps motion=1.00 presence=5.35 rssi=-33 + +Output JSON (per tick): + { + "ts": 1716830400.123, + "node": 0, # always 0 (single node), client expands to 4 + "motion": 1.00, # raw firmware metric + "presence": 5.35, + "rssi": -33, + "yield_pps": 15, + "amp": 0.78 # synthesized CSI amplitude in [0..1] for the bar + } + +Run on ruvultra: + python3 -u ruvultra-csi-bridge.py +""" +import asyncio +import builtins +import json +import re +import sys +import time +from contextlib import suppress + +# Force every print to flush — we're often piped to a log file +_orig_print = builtins.print +def _print(*a, **kw): + kw.setdefault("flush", True) + return _orig_print(*a, **kw) +builtins.print = _print + +import serial +import websockets + +PORT = "/dev/ttyACM0" +BAUD = 115200 +WS_HOST = "0.0.0.0" +WS_PORT = 8766 + +TICK_RE = re.compile( + r"adaptive_ctrl:\s*\w+\s+tick:\s*" + r"state=(?P\d+)\s+" + r"yield=(?P\d+)pps\s+" + r"motion=(?P[\d.]+)\s+" + r"presence=(?P[\d.]+)\s+" + r"rssi=(?P-?\d+)" +) + +clients = set() +last_payload = None + + +def amp_from_metrics(motion, presence, rssi): + """Map firmware metrics to a [0..1] CSI-style amplitude.""" + rssi_norm = max(0.0, min(1.0, (rssi + 80) / 50)) # -80..-30 → 0..1 + presence_norm = max(0.0, min(1.0, presence / 8.0)) # cap at 8 + motion_norm = max(0.0, min(1.0, motion)) # already 0..1ish + return 0.40 * rssi_norm + 0.35 * presence_norm + 0.25 * motion_norm + + +async def serial_reader_loop(): + global last_payload + print(f"[bridge] opening {PORT} @ {BAUD}…") + while True: + try: + ser = serial.Serial(PORT, BAUD, timeout=1) + except (serial.SerialException, OSError) as e: + print(f"[bridge] serial open failed ({e}); retry in 3s") + await asyncio.sleep(3) + continue + + print(f"[bridge] connected to {PORT}") + loop = asyncio.get_event_loop() + try: + while True: + line = await loop.run_in_executor(None, ser.readline) + if not line: + continue + try: + text = line.decode(errors="replace").strip() + except Exception: + continue + m = TICK_RE.search(text) + if not m: + continue + motion = float(m["motion"]) + presence = float(m["presence"]) + rssi = int(m["rssi"]) + payload = { + "ts": time.time(), + "node": 0, + "state": int(m["state"]), + "yield_pps": int(m["yield"]), + "motion": motion, + "presence": presence, + "rssi": rssi, + "amp": amp_from_metrics(motion, presence, rssi), + } + last_payload = payload + msg = json.dumps(payload) + if clients: + dead = [] + for ws in list(clients): + try: + await ws.send(msg) + except websockets.ConnectionClosed: + dead.append(ws) + for d in dead: + clients.discard(d) + print( + f"[tick] motion={motion:.2f} presence={presence:5.2f} " + f"rssi={rssi:+d} yield={int(m['yield']):3d}pps " + f"amp={payload['amp']:.2f} clients={len(clients)}" + ) + except (serial.SerialException, OSError) as e: + print(f"[bridge] serial error ({e}); reopen in 1s") + with suppress(Exception): + ser.close() + await asyncio.sleep(1) + + +async def ws_handler(ws): + addr = ws.remote_address + clients.add(ws) + print(f"[ws] client connected: {addr} total={len(clients)}") + try: + if last_payload is not None: + await ws.send(json.dumps(last_payload)) + await ws.wait_closed() + finally: + clients.discard(ws) + print(f"[ws] client gone: {addr} total={len(clients)}") + + +async def main(): + print(f"[bridge] websocket on ws://{WS_HOST}:{WS_PORT}") + async with websockets.serve(ws_handler, WS_HOST, WS_PORT): + await serial_reader_loop() + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + pass diff --git a/examples/three.js/server/serve-demo.py b/examples/three.js/server/serve-demo.py new file mode 100644 index 00000000..3ca9088e --- /dev/null +++ b/examples/three.js/server/serve-demo.py @@ -0,0 +1,46 @@ +"""Tiny threaded HTTP server for the three.js demos that fetch local files. + +Why a sibling helper script instead of `python -m http.server`? +The stdlib SimpleHTTPServer is single-threaded; Chrome opens many parallel +connections (HTML + 9 script tags + FBX), the first eats the worker, the +rest time out with net::ERR_EMPTY_RESPONSE. ThreadingHTTPServer fixes it. + +Usage: + python examples/three.js/server/serve-demo.py + open http://localhost:8765/examples/three.js/demos/05-skinned-realtime.html +""" +from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler +import os, sys + +PORT = int(os.environ.get("PORT", 8765)) +# Always serve from the repo root regardless of where the script is launched. +# This file lives at examples/three.js/server/serve-demo.py — three levels deep. +os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))) + +class NoCacheHandler(SimpleHTTPRequestHandler): + def end_headers(self): + # Aggressive no-cache so browser ALWAYS fetches the latest .html + # after we edit it. Otherwise stale code sticks around even on hard + # refresh and you debug a phantom. + self.send_header("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0") + self.send_header("Pragma", "no-cache") + self.send_header("Expires", "0") + super().end_headers() + +DEMOS = [ + "01-helpers.html", + "02-cinematic.html", + "03-skinned.html", + "04-skinned-fbx.html", + "05-skinned-realtime.html", +] + +with ThreadingHTTPServer(("127.0.0.1", PORT), NoCacheHandler) as srv: + print(f"serving {os.getcwd()} on http://127.0.0.1:{PORT}/") + print("demos:") + for d in DEMOS: + print(f" http://127.0.0.1:{PORT}/examples/three.js/demos/{d}") + try: + srv.serve_forever() + except KeyboardInterrupt: + sys.exit(0)