From deb561bf9c4f5f7f57210d55e4c772a470fa2dcb Mon Sep 17 00:00:00 2001 From: ruv Date: Tue, 12 May 2026 22:16:27 -0400 Subject: [PATCH] fix(rvcsi): scale-relative baseline-drift thresholds + ESP32 end-to-end validation BaselineDriftDetector compared `mean_amplitude` against its EWMA baseline with *absolute* thresholds (anomaly 1.0, drift 0.15). Fine for the synthetic unit tests (amplitudes ~1.0), but raw ESP32 CSI is int8 I/Q with amplitudes up to ~128, so window-to-window RMS distance is routinely 5-50 >> 1.0 and AnomalyDetected fired on ~96% of windows (319/331 on a real node-1 capture). Drift is now `||current - baseline||2 / ||baseline||2` (a fraction, with an eps floor that falls back to absolute for a degenerate near-zero baseline), so one tuning is valid across raw-int8 ESP32, int16-scaled Nexmon, and baseline-subtracted streams. AnomalyDetected drops to 40/331 on the same data; the existing detector tests still pass (their explicit configs are valid relative thresholds too); added baseline_drift_is_scale_invariant_ no_anomaly_storm. rvcsi-events 18 -> 19 tests; 162 rvcsi tests, 0 failures, clippy-clean. Surfaced by an end-to-end test against real ESP32 CSI on COM7: the device (ESP32-S3, node 1, ADR-018 firmware, WiFi "ruv.net" ch5 RSSI -39, CSI cb only because nothing listens at .156). rvcsi has no ESP32 adapter yet, so a 7,000-frame node-1 recording was transcoded to .rvcsi via the new scripts/esp32_jsonl_to_rvcsi.py (stand-in for `record --source esp32-jsonl`) and run through `rvcsi inspect`/`replay`/`calibrate`/`events` end-to-end. ADR-095 D13 and ADR-096 sections 2.1/5 updated; CHANGELOG entry added; rvcsi-adapter-esp32 (live serial/UDP source) noted as a follow-up. Co-Authored-By: claude-flow --- CHANGELOG.md | 18 +- .../ADR-095-rvcsi-edge-rf-sensing-platform.md | 2 +- docs/adr/ADR-096-rvcsi-ffi-crate-layout.md | 9 +- scripts/esp32_jsonl_to_rvcsi.py | 188 ++++++++++++++++++ v2/crates/rvcsi-events/src/detectors.rs | 89 ++++++++- 5 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 scripts/esp32_jsonl_to_rvcsi.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 12ad795c..36731afc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,12 +17,26 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ADR-079 target (35%+), and notes the eval phases are pending. Surfaced by the PowerPlatePulse training-pipeline audit (2026-05-11); 6 remaining audit findings tracked in the PR. +- **rvCSI `BaselineDriftDetector`: drift thresholds are now scale-relative, not absolute.** + The detector compared `mean_amplitude` against its EWMA baseline with absolute + thresholds (`anomaly_threshold = 1.0`, `drift_threshold = 0.15`) — fine for the + synthetic unit tests (amplitudes ≈ 1.0), but raw ESP32 CSI is `int8` I/Q with + amplitudes up to ~128, so the window-to-window RMS distance is routinely 5–50 ≫ 1.0 + and `AnomalyDetected` fired on ~96 % of windows (319/331 on a real node-1 capture). + Drift is now `‖current − baseline‖₂ / ‖baseline‖₂` (a fraction, with an `eps` floor + for a degenerate near-zero baseline), so one tuning works across raw-`int8` ESP32, + `int16`-scaled Nexmon, and baseline-subtracted streams alike — `AnomalyDetected` + drops to 40/331 on the same data, the existing detector tests still pass, and a + `baseline_drift_is_scale_invariant_no_anomaly_storm` regression test was added. + ADR-095 D13 / ADR-096 §2.1, §5 updated. Surfaced by an end-to-end test against + real ESP32 CSI (a 7,000-frame node-1 capture; transcoder at + `scripts/esp32_jsonl_to_rvcsi.py`). ### Added - **rvCSI — edge RF sensing runtime (design + first implementation).** New subsystem **rvCSI**: a Rust-first / TypeScript-accessible / hardware-abstracted edge RF sensing runtime that normalizes WiFi CSI from Nexmon, ESP32, Intel, Atheros, file and replay sources into one validated `CsiFrame` schema, runs reusable DSP, emits typed confidence-scored events, and bridges to RuVector RF memory, an MCP tool server and a TS SDK. - **Design docs:** `docs/prd/rvcsi-platform-prd.md` (purpose, users, success criteria, FR1–FR10, NFRs, system architecture, data model); `docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md` (the 15 architectural decisions: Rust core, C-at-the-boundary, TS SDK via napi-rs, normalized schema, validate-before-FFI, CSI-as-temporal-delta, RuVector as RF memory, replayability, detection≠decision, local-first, read-first/write-gated MCP, mandatory quality scoring, versioned calibration, plugin adapters); `docs/adr/ADR-096-rvcsi-ffi-crate-layout.md` (crate topology, the napi-c shim record format & contract, the napi-rs Node surface, build/test invariants); `docs/ddd/rvcsi-domain-model.md` (7 bounded contexts: Capture, Validation, Signal, Calibration, Event, Memory, Agent — with aggregates, invariants, context map and domain services). Indexed in `docs/adr/README.md` and `docs/ddd/README.md`. - - **Crates** (9 new `v2/crates/rvcsi-*` workspace members): `rvcsi-core` (normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, `AdapterProfile`, `CsiSource` plugin trait, id newtypes + `IdGenerator`, `RvcsiError`, the `validate_frame` pipeline + quality scoring; `forbid(unsafe_code)`); `rvcsi-adapter-nexmon` — the **napi-c** seam: `native/rvcsi_nexmon_shim.{c,h}` (the only C in the runtime — allocation-free, bounds-checked, ABI `1.1`), compiled via `build.rs`+`cc`, handling **two byte formats** — the compact self-describing "rvCSI Nexmon record", and the **real nexmon_csi UDP payload** (the 18-byte `magic 0x1111 · rssi · fctl · src_mac · seq · core/stream · chanspec · chip_ver` header + `nsub` int16 I/Q samples, the modern BCM43455c0/4358/4366c0 export read by CSIKit/`csireader.py`), with a Broadcom d11ac **chanspec decoder** (channel/bandwidth/band) — plus a pure-Rust **libpcap reader** (classic `.pcap`, all byte-order/timestamp-resolution magics, Ethernet/raw-IPv4/Linux-SLL link types) and a **Nexmon-chip / Raspberry-Pi-model registry** (`NexmonChip` / `RaspberryPiModel` — including the **Raspberry Pi 5** (CYW43455/BCM43455c0, same wireless as the Pi 4 — 20/40/80 MHz, 2.4+5 GHz, 64/128/256 subcarriers), the Pi 3B+/4/400, and the Pi Zero 2 W (BCM43436b0); `nexmon_adapter_profile` / `raspberry_pi_profile` build the per-chip `AdapterProfile`; `chip_ver` words auto-resolve to a chip). Wrapped by a documented `ffi` module and two `CsiSource`s: `NexmonAdapter` (record buffers) and `NexmonPcapAdapter` (real nexmon_csi UDP inside a `tcpdump -i wlan0 dst port 5500 -w csi.pcap` capture — the pcap timestamp stamps each frame; the chip is auto-detected from `chip_ver`, overridable via `.with_pi_model(Pi5)` / `.with_chip(...)`). `rvcsi-dsp` (DC removal, phase unwrap, smoothing, Hampel/MAD filter, sliding variance, baseline subtraction, motion-energy/presence/confidence features, heuristic breathing-band estimate, non-destructive `SignalPipeline`); `rvcsi-events` (`WindowBuffer`, the `EventDetector` trait + presence/motion/quality/baseline-drift state machines, `EventPipeline`); `rvcsi-adapter-file` (the `.rvcsi` JSONL capture format, `FileRecorder`, `FileReplayAdapter` deterministic replay); `rvcsi-ruvector` (deterministic window/event embeddings, `cosine_similarity`, the `RfMemoryStore` trait, `InMemoryRfMemory` + `JsonlRfMemory` — a standin until the production RuVector binding); `rvcsi-runtime` (the no-FFI composition layer: `CaptureRuntime` = `CsiSource` + `validate_frame` + `SignalPipeline` + `EventPipeline`, plus one-shot helpers `summarize_capture`/`decode_nexmon_records`/`decode_nexmon_pcap`/`summarize_nexmon_pcap`/`events_from_capture`/`export_capture_to_rf_memory`); `rvcsi-node` — the **napi-rs** seam (a `["cdylib","rlib"]` Node addon, `build.rs` runs `napi_build::setup()`; thin `#[napi]` wrappers over `rvcsi-runtime` — `nexmonDecodeRecords`/`nexmonDecodePcap` (with optional `chip`)/`inspectNexmonPcap`/`decodeChanspec`/`nexmonChipName`/`nexmonProfile`/`nexmonChips`/`inspectCaptureFile`/`eventsFromCaptureFile`/`exportCaptureToRfMemory` + an `RvcsiRuntime` streaming class; everything that crosses to JS is a validated/normalized struct serialized to JSON); `rvcsi-cli` (the `rvcsi` binary: `record` (Nexmon-dump *or* `--source nexmon-pcap [--chip pi5]` → `.rvcsi`), `inspect`, `inspect-nexmon`, `nexmon-chips`, `decode-chanspec`, `replay`, `stream`, `events`, `health`, `calibrate` v0-baseline, `export ruvector`). Plus the `@ruv/rvcsi` npm package (`package.json`/`index.js`/`index.d.ts`/`README`/`__test__`) alongside `rvcsi-node` — a curated JS surface that parses the addon's JSON into plain `CsiFrame`/`CsiWindow`/`CsiEvent`/`SourceHealth`/`CaptureSummary`/`NexmonPcapSummary`/`DecodedChanspec` objects, with a lazy native-addon load. - - **Tests:** 168 across the rvcsi crates (core 29, dsp 28, events 18, adapter-file 20 + 1 doctest, adapter-nexmon 28 — round-tripping through the C shim and synthetic libpcap files, incl. Pi 5 / chip-detection, ruvector 20 + 1 doctest, runtime 13, cli 10), 0 failures; all rvcsi crates build together and are clippy-clean (`rvcsi-node` under `deny(clippy::all)`); `forbid(unsafe_code)` everywhere except `rvcsi-adapter-nexmon` (FFI, every `unsafe` block documented). Not yet wired in: live radio capture, the WebSocket daemon (`rvcsi-daemon`), the MCP tool server (`rvcsi-mcp`), and the legacy nexmon *packed-float* CSI export — follow-ups on top of these crates. + - **Crates** (9 new `v2/crates/rvcsi-*` workspace members): `rvcsi-core` (normalized `CsiFrame`/`CsiWindow`/`CsiEvent` schema, `AdapterProfile`, `CsiSource` plugin trait, id newtypes + `IdGenerator`, `RvcsiError`, the `validate_frame` pipeline + quality scoring; `forbid(unsafe_code)`); `rvcsi-adapter-nexmon` — the **napi-c** seam: `native/rvcsi_nexmon_shim.{c,h}` (the only C in the runtime — allocation-free, bounds-checked, ABI `1.1`), compiled via `build.rs`+`cc`, handling **two byte formats** — the compact self-describing "rvCSI Nexmon record", and the **real nexmon_csi UDP payload** (the 18-byte `magic 0x1111 · rssi · fctl · src_mac · seq · core/stream · chanspec · chip_ver` header + `nsub` int16 I/Q samples, the modern BCM43455c0/4358/4366c0 export read by CSIKit/`csireader.py`), with a Broadcom d11ac **chanspec decoder** (channel/bandwidth/band) — plus a pure-Rust **libpcap reader** (classic `.pcap`, all byte-order/timestamp-resolution magics, Ethernet/raw-IPv4/Linux-SLL link types) and a **Nexmon-chip / Raspberry-Pi-model registry** (`NexmonChip` / `RaspberryPiModel` — including the **Raspberry Pi 5** (CYW43455/BCM43455c0, same wireless as the Pi 4 — 20/40/80 MHz, 2.4+5 GHz, 64/128/256 subcarriers), the Pi 3B+/4/400, and the Pi Zero 2 W (BCM43436b0); `nexmon_adapter_profile` / `raspberry_pi_profile` build the per-chip `AdapterProfile`; `chip_ver` words auto-resolve to a chip). Wrapped by a documented `ffi` module and two `CsiSource`s: `NexmonAdapter` (record buffers) and `NexmonPcapAdapter` (real nexmon_csi UDP inside a `tcpdump -i wlan0 dst port 5500 -w csi.pcap` capture — the pcap timestamp stamps each frame; the chip is auto-detected from `chip_ver`, overridable via `.with_pi_model(Pi5)` / `.with_chip(...)`). `rvcsi-dsp` (DC removal, phase unwrap, smoothing, Hampel/MAD filter, sliding variance, baseline subtraction, motion-energy/presence/confidence features, heuristic breathing-band estimate, non-destructive `SignalPipeline`); `rvcsi-events` (`WindowBuffer`, the `EventDetector` trait + presence/motion/quality/baseline-drift state machines, `EventPipeline`; the baseline-drift detector uses **scale-relative** thresholds — drift as a fraction of the baseline's RMS magnitude — so one tuning works across raw-`int8` ESP32, `int16`-scaled Nexmon, and baseline-subtracted streams alike); `rvcsi-adapter-file` (the `.rvcsi` JSONL capture format, `FileRecorder`, `FileReplayAdapter` deterministic replay); `rvcsi-ruvector` (deterministic window/event embeddings, `cosine_similarity`, the `RfMemoryStore` trait, `InMemoryRfMemory` + `JsonlRfMemory` — a standin until the production RuVector binding); `rvcsi-runtime` (the no-FFI composition layer: `CaptureRuntime` = `CsiSource` + `validate_frame` + `SignalPipeline` + `EventPipeline`, plus one-shot helpers `summarize_capture`/`decode_nexmon_records`/`decode_nexmon_pcap`/`summarize_nexmon_pcap`/`events_from_capture`/`export_capture_to_rf_memory`); `rvcsi-node` — the **napi-rs** seam (a `["cdylib","rlib"]` Node addon, `build.rs` runs `napi_build::setup()`; thin `#[napi]` wrappers over `rvcsi-runtime` — `nexmonDecodeRecords`/`nexmonDecodePcap` (with optional `chip`)/`inspectNexmonPcap`/`decodeChanspec`/`nexmonChipName`/`nexmonProfile`/`nexmonChips`/`inspectCaptureFile`/`eventsFromCaptureFile`/`exportCaptureToRfMemory` + an `RvcsiRuntime` streaming class; everything that crosses to JS is a validated/normalized struct serialized to JSON); `rvcsi-cli` (the `rvcsi` binary: `record` (Nexmon-dump *or* `--source nexmon-pcap [--chip pi5]` → `.rvcsi`), `inspect`, `inspect-nexmon`, `nexmon-chips`, `decode-chanspec`, `replay`, `stream`, `events`, `health`, `calibrate` v0-baseline, `export ruvector`). Plus the `@ruv/rvcsi` npm package (`package.json`/`index.js`/`index.d.ts`/`README`/`__test__`) alongside `rvcsi-node` — a curated JS surface that parses the addon's JSON into plain `CsiFrame`/`CsiWindow`/`CsiEvent`/`SourceHealth`/`CaptureSummary`/`NexmonPcapSummary`/`DecodedChanspec` objects, with a lazy native-addon load. + - **Tests:** 169 across the rvcsi crates (core 29, dsp 28, events 19 — incl. a baseline-drift scale-invariance regression, adapter-file 20 + 1 doctest, adapter-nexmon 28 — round-tripping through the C shim and synthetic libpcap files, incl. Pi 5 / chip-detection, ruvector 20 + 1 doctest, runtime 13, cli 10), 0 failures; all rvcsi crates build together and are clippy-clean (`rvcsi-node` under `deny(clippy::all)`); `forbid(unsafe_code)` everywhere except `rvcsi-adapter-nexmon` (FFI, every `unsafe` block documented). Also exercised end-to-end against a real 7,000-frame ESP32 node-1 capture (transcoded with `scripts/esp32_jsonl_to_rvcsi.py` — the stand-in for the not-yet-shipped `record --source esp32-jsonl`): `rvcsi inspect`/`replay`/`calibrate`/`events` all run on real hardware data. Not yet wired in: live radio capture, `rvcsi-adapter-esp32` (live serial/UDP ESP32 source), the WebSocket daemon (`rvcsi-daemon`), the MCP tool server (`rvcsi-mcp`), and the legacy nexmon *packed-float* CSI export — follow-ups on top of these crates. - **`wifi-densepose-train`: `signal_features` module — wires `wifi-densepose-signal` into the training pipeline.** `wifi-densepose-signal` was previously a phantom dependency of `wifi-densepose-train` (listed in `Cargo.toml`, never imported). New `wifi_densepose_train::signal_features::extract_signal_features` (and `CsiSample::signal_features()`) run a windowed CSI observation's centre frame through `wifi_densepose_signal::features::FeatureExtractor`, producing a fixed-length (`FEATURE_LEN = 12`) amplitude/phase/PSD feature vector — the hook for a future vitals / multi-task supervision head (breathing- and heart-rate-band power are read off the PSD summary). The vector is produced on demand and not yet fed back into the loss. Surfaced by the 2026-05-11 training-pipeline audit (findings #1 "vitals features absent from training" and #2 "`wifi-densepose-signal` ghost dep"). - **`wifi-densepose-train`: `TrainingConfig` subcarrier-layout presets + a real-loader integration test.** New `TrainingConfig::for_subcarriers(native, target)` plus named presets `ht40_192()` (≈192-sc ESP32 HT40 → 56) and `multiband_168()` (168-sc ADR-078 multi-band mesh → 56), so non-MM-Fi CSI shapes are first-class instead of requiring manual `native_subcarriers`/`num_subcarriers` overrides; field docs now list the supported source counts and the multi-NIC mapping. New `tests/test_real_loader.rs` round-trips synthetic CSI through `.npy` files → `MmFiDataset::discover`/`get` (including the subcarrier-interpolation branch and the empty-root case) — exercising the on-disk loader path the deterministic `verify-training` proof intentionally bypasses. Addresses training-pipeline audit findings #6 (56-sc/1-NIC config default) and #7 (multi-band mesh not in config); the #4 concern ("proof uses synthetic data") is reframed — the proof *should* use a reproducible source, and this test covers the real loader it skips. diff --git a/docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md b/docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md index 030458c7..101502dc 100644 --- a/docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md +++ b/docs/adr/ADR-095-rvcsi-edge-rf-sensing-platform.md @@ -111,7 +111,7 @@ Agents should observe RF state safely; device mutation and calibration change sy ### D13 — Quality scoring is mandatory CSI quality varies widely by chip, antenna, environment, channel, and interference. **Every frame, window, and event carries quality or confidence scoring.** -*Consequences:* downstream systems can suppress weak evidence; easier debugging; requires calibration and thresholds. +*Consequences:* downstream systems can suppress weak evidence; easier debugging; requires calibration and thresholds. Where a detector compares against a learned baseline (e.g. baseline-drift / anomaly), thresholds are expressed **relative to the baseline's magnitude**, not as absolute amplitude units, so a single tuning is valid across sources whose raw CSI scales differ by orders of magnitude (raw `int8` ESP32 vs. `int16`-scaled Nexmon vs. baseline-subtracted streams). ### D14 — Versioned calibration profiles diff --git a/docs/adr/ADR-096-rvcsi-ffi-crate-layout.md b/docs/adr/ADR-096-rvcsi-ffi-crate-layout.md index cc4bc31b..794f0934 100644 --- a/docs/adr/ADR-096-rvcsi-ffi-crate-layout.md +++ b/docs/adr/ADR-096-rvcsi-ffi-crate-layout.md @@ -36,7 +36,7 @@ Eight new workspace members under `v2/crates/`: |-------|-----------|------------|------| | `rvcsi-core` | no (`forbid`) | — (serde, thiserror) | The normalized schema (`CsiFrame`/`CsiWindow`/`CsiEvent`), `AdapterProfile`, the `CsiSource` plugin trait, id newtypes + `IdGenerator`, `RvcsiError`, and the `validate_frame` pipeline + quality scoring. The shared kernel. | | `rvcsi-dsp` | no (`forbid`) | `rvcsi-core` | Reusable DSP stages (DC removal, phase unwrap, smoothing, Hampel/MAD outlier filter, sliding variance, baseline subtraction) and scalar features (motion energy, presence score, confidence, heuristic breathing-band estimate), plus a non-destructive `SignalPipeline::process_frame`. | -| `rvcsi-events` | no (`forbid`) | `rvcsi-core` | `WindowBuffer` (frames → `CsiWindow`), the `EventDetector` trait + presence/motion/quality/baseline-drift state machines, and `EventPipeline` (windows → `CsiEvent`s). | +| `rvcsi-events` | no (`forbid`) | `rvcsi-core` | `WindowBuffer` (frames → `CsiWindow`), the `EventDetector` trait + presence/motion/quality/baseline-drift state machines, and `EventPipeline` (windows → `CsiEvent`s). The baseline-drift detector measures drift **relative to the running baseline's RMS magnitude** (a fraction, not absolute amplitude units), so the same thresholds work for raw `int8` ESP32 CSI, `int16`-scaled Nexmon CSI, and baseline-subtracted streams alike — see ADR-095 D13. | | `rvcsi-adapter-file` | no (`forbid`) | `rvcsi-core` | The `.rvcsi` capture format (JSONL: a header line + one `CsiFrame` per line), `FileRecorder`, and `FileReplayAdapter` (a `CsiSource`) — deterministic replay (D9). | | `rvcsi-adapter-nexmon` | **yes** (FFI only) | `rvcsi-core` + the C shim | The **napi-c** seam: `native/rvcsi_nexmon_shim.{c,h}` compiled via `build.rs`+`cc`, a documented `ffi` module wrapping it, a pure-Rust libpcap reader (`pcap.rs`), the Nexmon-chip / Raspberry-Pi-model registry (`chips.rs` — `NexmonChip`, `RaspberryPiModel` incl. **Pi 5**, profile builders), and two `CsiSource`s — `NexmonAdapter` (rvCSI-record buffers) and `NexmonPcapAdapter` (real nexmon_csi UDP payloads inside a `.pcap`, with chip auto-detection). | | `rvcsi-ruvector` | no (`forbid`) | `rvcsi-core` | The RuVector RF-memory bridge: deterministic `window_embedding`/`event_embedding`, `cosine_similarity`, the `RfMemoryStore` trait, and `InMemoryRfMemory` + `JsonlRfMemory` (a standin until the production RuVector binding lands). | @@ -126,11 +126,12 @@ A real deployment captures with `tcpdump -i wlan0 dst port 5500 -w csi.pcap` on - `rvcsi-core` — implemented, `forbid(unsafe_code)`, 29 unit tests. - `rvcsi-adapter-nexmon` + the napi-c shim — implemented; C (ABI `1.1`) compiled via `build.rs`+`cc`; the `ffi` module wraps both record formats (rvCSI record **and** the real nexmon_csi UDP payload + chanspec decode); a pure-Rust `pcap` reader; the Nexmon-chip / Raspberry-Pi-model registry (`chips.rs` — incl. **Pi 5 → BCM43455c0** + chip auto-detection from `chip_ver`); `NexmonAdapter` + `NexmonPcapAdapter` `CsiSource`s; 28 tests, several round-tripping through the C shim and through synthetic libpcap files. -- `rvcsi-dsp` (28 tests), `rvcsi-events` (18 tests), `rvcsi-adapter-file` (20 + 1 doctest), `rvcsi-ruvector` (20 + 1 doctest) — implemented. +- `rvcsi-dsp` (28 tests), `rvcsi-events` (19 tests — incl. a scale-invariance regression for the baseline-drift detector), `rvcsi-adapter-file` (20 + 1 doctest), `rvcsi-ruvector` (20 + 1 doctest) — implemented. - `rvcsi-runtime` (13 tests) — composition layer + the one-shot helpers, including `decode_nexmon_pcap` / `decode_nexmon_pcap_for` (per-chip) / `summarize_nexmon_pcap` / `nexmon_profile_for`. - `rvcsi-node` (napi-rs surface — incl. `nexmonDecodePcap` (with `chip`) / `inspectNexmonPcap` / `decodeChanspec` / `nexmonChipName` / `nexmonProfile` / `nexmonChips` / `RvcsiRuntime.openNexmonPcap`) and `rvcsi-cli` (10 tests — incl. `record --source nexmon-pcap [--chip pi5]`, `inspect-nexmon`, `nexmon-chips`, `decode-chanspec`) — implemented; the `@ruv/rvcsi` npm package + a Node smoke test ship alongside. -- Totals: 168 rvcsi unit/integration tests + 2 doctests, 0 failures; all rvcsi crates build together and are clippy-clean. -- `rvcsi-mcp` (MCP tool server), `rvcsi-daemon` (live capture + WebSocket), and the legacy nexmon *packed-float* CSI export — not in this PR; tracked as follow-ups. +- Totals: 169 rvcsi unit/integration tests + 2 doctests, 0 failures; all rvcsi crates build together and are clippy-clean. +- **Validated against real ESP32 CSI** (a 7,000-frame node-1 capture, transcoded to `.rvcsi` via `scripts/esp32_jsonl_to_rvcsi.py` — the stand-in for the not-yet-shipped `record --source esp32-jsonl`): `rvcsi inspect` / `replay` / `calibrate` / `events` all run end-to-end. This surfaced and fixed the baseline-drift over-trigger (absolute → relative thresholds, above). +- `rvcsi-adapter-esp32` (live serial/UDP ESP32 source — ADR-095 §1.2 / D15), `rvcsi-mcp` (MCP tool server), `rvcsi-daemon` (live capture + WebSocket), and the legacy nexmon *packed-float* CSI export — not in this PR; tracked as follow-ups. --- diff --git a/scripts/esp32_jsonl_to_rvcsi.py b/scripts/esp32_jsonl_to_rvcsi.py new file mode 100644 index 00000000..759955f5 --- /dev/null +++ b/scripts/esp32_jsonl_to_rvcsi.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Transcode an ESP32 .csi.jsonl recording into a .rvcsi capture (JSONL). + +This is the moral equivalent of `rvcsi record --source esp32-jsonl` (which the +PR does not ship yet): parse each ESP32 frame, derive amplitude/phase from the +raw int8 I/Q pairs, run the same validation/quality logic rvcsi_core does, and +write a .rvcsi file whose first line is a CaptureHeader and every later line a +CsiFrame. Rejected frames are dropped (quarantine), like the real pipeline. + +Usage: esp32_jsonl_to_rvcsi.py [--limit N] +""" +import json +import math +import sys + +# --- rvcsi_core::ValidationPolicy::default() ------------------------------- +MIN_SUBCARRIERS = 1 +MAX_SUBCARRIERS = 4096 +RSSI_LO, RSSI_HI = -110, 0 +MIN_QUALITY = 0.25 +RSSI_HARD_MARGIN = 30 + + +def quality_and_status(amplitude, rssi_dbm): + """Faithful port of rvcsi_core::validation::validate_frame soft scoring.""" + reasons = [] + q = 1.0 + sc = len(amplitude) + # out-of-range (non-fatal) RSSI + if rssi_dbm is not None and (rssi_dbm < RSSI_LO or rssi_dbm > RSSI_HI): + q *= 0.6 + reasons.append(f"rssi {rssi_dbm} dBm outside [{RSSI_LO},{RSSI_HI}]") + # dead subcarriers + dead = sum(1 for a in amplitude if a < 1e-6) + if dead > 0: + frac = dead / max(sc, 1) + q *= max(1.0 - frac, 0.05) + reasons.append(f"{dead}/{sc} dead subcarriers") + # amplitude spike vs median + if sc >= 3: + s = sorted(amplitude) + median = max(s[sc // 2], 1e-9) + mx = s[-1] + if mx > median * 50.0: + q *= 0.7 + reasons.append(f"amplitude spike: max {mx:.3f} vs median {median:.3f}") + if rssi_dbm is None: + q *= 0.95 + reasons.append("missing rssi") + q = min(max(q, 0.0), 1.0) + if q < MIN_QUALITY: + status = "Degraded" # degrade_instead_of_reject = true + else: + status = "Accepted" + return q, status, reasons + + +def main(): + if len(sys.argv) < 3: + print(__doc__) + sys.exit(2) + in_path, out_path = sys.argv[1], sys.argv[2] + limit = None + if "--limit" in sys.argv: + limit = int(sys.argv[sys.argv.index("--limit") + 1]) + + source_id = "esp32-com7-rec" + header = { + "rvcsi_capture_version": 1, + "session_id": 0, + "source_id": source_id, + "adapter_profile": { + "adapter_kind": "Esp32", + "chip": "ESP32-S3", + "firmware_version": None, + "driver_version": None, + "supported_channels": [], + "supported_bandwidths_mhz": [], + "expected_subcarrier_counts": [], + "supports_live_capture": True, + "supports_injection": False, + "supports_monitor_mode": False, + }, + "validation_policy": { + "min_subcarriers": MIN_SUBCARRIERS, + "max_subcarriers": MAX_SUBCARRIERS, + "rssi_dbm_bounds": [RSSI_LO, RSSI_HI], + "strict_monotonic_time": False, + "degrade_instead_of_reject": True, + "min_quality": MIN_QUALITY, + }, + "calibration_version": None, + "runtime_config_json": "{}", + "created_unix_ns": 0, + } + + stats = { + "read": 0, "written": 0, + "rej_len": 0, "rej_sc": 0, "rej_nonfinite": 0, "rej_rssi": 0, + "accepted": 0, "degraded": 0, + } + sc_hist = {} + out = open(out_path, "w", newline="\n") + out.write(json.dumps(header, separators=(",", ":")) + "\n") + fid = 0 + with open(in_path) as f: + for line in f: + line = line.strip() + if not line: + continue + d = json.loads(line) + if d.get("type") != "raw_csi": + continue + stats["read"] += 1 + if limit is not None and stats["read"] > limit: + stats["read"] -= 1 + break + iq_hex = d.get("iq_hex", "") + raw = bytes.fromhex(iq_hex) + n_pairs = len(raw) // 2 + # ESP-IDF CSI buffer layout: [imag0, real0, imag1, real1, ...] as int8 + i_vals, q_vals, amp, ph = [], [], [], [] + for k in range(n_pairs): + imag = raw[2 * k] + real = raw[2 * k + 1] + if imag >= 128: + imag -= 256 + if real >= 128: + real -= 256 + fi, fq = float(real), float(imag) + i_vals.append(fi) + q_vals.append(fq) + amp.append(math.sqrt(fi * fi + fq * fq)) + ph.append(math.atan2(fq, fi)) + sc = n_pairs + sc_hist[sc] = sc_hist.get(sc, 0) + 1 + # hard checks (mirror validate_frame) + if sc < MIN_SUBCARRIERS or sc > MAX_SUBCARRIERS: + stats["rej_sc"] += 1 + continue + # int8 -> always finite, lengths consistent by construction + # RSSI: the v1 collector's rssi byte is unreliable (sentinels 64/-128 + # etc.); only carry it through when it lands in a plausible band, + # otherwise leave it None (a small quality penalty, not a reject). + r = d.get("rssi") + rssi_dbm = r if (isinstance(r, int) and -140 <= r <= 30) else None + if rssi_dbm is not None and (rssi_dbm < RSSI_LO - RSSI_HARD_MARGIN or rssi_dbm > RSSI_HI + RSSI_HARD_MARGIN): + stats["rej_rssi"] += 1 + continue + if rssi_dbm is not None and not (-110 <= rssi_dbm <= 0): + rssi_dbm = None # implausible but not insane -> drop the field + q, status, reasons = quality_and_status(amp, rssi_dbm) + ch = d.get("channel", 0) or 0 + frame = { + "frame_id": fid, + "session_id": 0, + "source_id": source_id, + "adapter_kind": "Esp32", + "timestamp_ns": int(d.get("ts_ns", 0)), + "channel": int(ch), + "bandwidth_mhz": 20, + "rssi_dbm": rssi_dbm, + "noise_floor_dbm": None, + "antenna_index": 0, + "tx_chain": None, + "rx_chain": None, + "subcarrier_count": sc, + "i_values": i_vals, + "q_values": q_vals, + "amplitude": amp, + "phase": ph, + "validation": status, + "quality_score": q, + } + if reasons: + frame["quality_reasons"] = reasons + frame["calibration_version"] = None + out.write(json.dumps(frame, separators=(",", ":")) + "\n") + fid += 1 + stats["written"] += 1 + stats[status.lower()] = stats.get(status.lower(), 0) + 1 + out.close() + print("transcode stats:", json.dumps(stats)) + print("subcarrier-count histogram:", json.dumps(dict(sorted(sc_hist.items(), key=lambda x: -x[1])))) + + +if __name__ == "__main__": + main() diff --git a/v2/crates/rvcsi-events/src/detectors.rs b/v2/crates/rvcsi-events/src/detectors.rs index 43d87766..4df12521 100644 --- a/v2/crates/rvcsi-events/src/detectors.rs +++ b/v2/crates/rvcsi-events/src/detectors.rs @@ -425,15 +425,24 @@ impl EventDetector for QualityDetector { // --------------------------------------------------------------------------- /// Tunables for [`BaselineDriftDetector`]. +/// +/// `drift_threshold` and `anomaly_threshold` are **relative** — they are +/// fractions of the running baseline's RMS magnitude, not absolute amplitude +/// units. This keeps the detector source-agnostic: ESP32 emits raw `int8` I/Q +/// (amplitudes up to ~128), Nexmon emits `int16`-scaled CSI, and a +/// baseline-subtracted pipeline emits values near zero — an *absolute* threshold +/// can only ever be right for one of them, a *relative* one is right for all. #[derive(Debug, Clone, Copy, PartialEq)] pub struct BaselineDriftConfig { - /// Per-window drift `||mean_amplitude - baseline||_2 / sqrt(n)` above this - /// for `drift_windows` windows in a row triggers [`CsiEventKind::BaselineChanged`]. + /// Relative per-window drift `||mean_amplitude - baseline||_2 / ||baseline||_2` + /// above this for `drift_windows` windows in a row triggers + /// [`CsiEventKind::BaselineChanged`]. `0.15` ≈ "the room moved ~15 %". pub drift_threshold: f32, /// Consecutive drifting windows before [`CsiEventKind::BaselineChanged`] fires. pub drift_windows: u32, - /// A single window with drift above this (much larger) value triggers - /// [`CsiEventKind::AnomalyDetected`]. + /// A single window whose relative drift exceeds this (much larger) value + /// triggers [`CsiEventKind::AnomalyDetected`]. `1.0` ≈ "this window differs + /// from the baseline by as much as the baseline's own magnitude". pub anomaly_threshold: f32, /// EWMA smoothing factor for the running baseline (`baseline = a*current + (1-a)*baseline`). pub ewma_alpha: f32, @@ -503,6 +512,37 @@ impl BaselineDriftDetector { (sq.sqrt() / (n as f64).sqrt()) as f32 } + /// Root-mean-square magnitude of a vector (`0.0` for an empty one). + fn rms(v: &[f32]) -> f32 { + let n = v.len(); + if n == 0 { + return 0.0; + } + let sq: f64 = v.iter().map(|&x| (x as f64) * (x as f64)).sum(); + (sq.sqrt() / (n as f64).sqrt()) as f32 + } + + /// Drift of `current` from `baseline` as a fraction of the baseline's RMS + /// magnitude. Source-agnostic (see [`BaselineDriftConfig`]). The `eps` floor + /// keeps a near-zero baseline (e.g. just after a baseline-subtraction stage) + /// from blowing the ratio up to infinity — when the baseline carries + /// essentially no energy there is nothing to drift *relative to*, so the + /// detector treats it as quiet. + fn relative_drift(current: &[f32], baseline: &[f32]) -> f32 { + let abs_drift = Self::rms_distance(current, baseline); + let baseline_rms = Self::rms(baseline); + // 1e-3 is well below any real CSI amplitude scale (ESP32 int8 ⇒ O(10), + // Nexmon int16 ⇒ O(100s)) yet above f32 noise. + const EPS: f32 = 1e-3; + if baseline_rms <= EPS { + // Degenerate baseline: fall back to an absolute reading so a sudden + // jump away from a flat-zero baseline still registers. + abs_drift + } else { + abs_drift / baseline_rms + } + } + fn update_ewma(&mut self, current: &[f32]) { match &mut self.baseline { None => self.baseline = Some(current.to_vec()), @@ -537,7 +577,7 @@ impl EventDetector for BaselineDriftDetector { Some(b) => b.clone(), }; - let drift = Self::rms_distance(current, &baseline); + let drift = Self::relative_drift(current, &baseline); let mut out = Vec::new(); if drift > self.cfg.anomaly_threshold { @@ -767,6 +807,45 @@ mod tests { } } + #[test] + fn baseline_drift_is_scale_invariant_no_anomaly_storm() { + // Regression for the ESP32 live-capture finding: raw int8 CSI amplitudes + // are O(10–128), so an *absolute* anomaly_threshold of 1.0 fired on + // essentially every window. With a *relative* threshold a few-percent + // wobble around a large baseline must stay quiet. + let g = IdGenerator::new(); + let mut d = BaselineDriftDetector::new(); // defaults: drift 0.15, anomaly 1.0 + // A realistic ESP32-ish window: two big "DC/pilot" subcarriers plus a + // band of small data subcarriers; ±3 % jitter window to window. + let base: Vec = { + let mut v = vec![128.0, 110.0]; + v.extend(std::iter::repeat(15.0).take(68)); + v + }; + let mut events = Vec::new(); + for k in 0..40u64 { + // deterministic small wobble in [-0.03, +0.03] * value + let f = 1.0 + 0.03 * (((k * 2654435761) % 7) as f32 / 3.0 - 1.0); + let w: Vec = base.iter().map(|x| x * f).collect(); + events.extend(d.on_window(&window_amp(k, (k + 1) * 1_000, w), &g)); + } + assert!( + !events.iter().any(|e| e.kind == CsiEventKind::AnomalyDetected), + "a ±3% wobble around a large baseline must not be an anomaly; got {events:?}" + ); + // A 5x jump on the data subcarriers (a person walks in) *is* an anomaly. + let spike: Vec = { + let mut v = vec![128.0, 110.0]; + v.extend(std::iter::repeat(75.0).take(68)); + v + }; + let ev = d.on_window(&window_amp(99, 100_000, spike), &g); + assert!( + ev.iter().any(|e| e.kind == CsiEventKind::AnomalyDetected), + "a 5x jump on the data band should register; got {ev:?}" + ); + } + #[test] fn baseline_drift_resets_on_subcarrier_change() { let g = IdGenerator::new();