test(python): close the three parity-review findings (native anchor in CI, NaN, cosine)
A cross-vendor review of the parity rework (2febbb81) found three issues; all
verified and fixed here.
1. HIGH — the native≈golden half never ran in CI, so a binding marshalling bug
could pass. The golden vectors were regenerated through the Python binding,
and only pytest (binding vs that golden) runs in CI — the native reference
tests in `python/tests/aether_parity.rs` link against the PyO3 crate and no
workflow runs them. A stable PyO3 conversion defect present at regeneration
would therefore be baked into the golden and go undetected.
Fix: a real native parity test IN the `wifi-densepose-aether` crate
(`tests/golden_parity.rs`), which is std-only and a member of the v2
workspace, so it runs under the existing `cargo test --workspace`. It
recomputes the embedding with no PyO3/marshalling and asserts it matches the
SAME committed golden within tolerance. Now native≈golden AND binding≈golden
both run in CI ⇒ binding≈native, and a binding-specific artifact in the
golden surfaces here as a native mismatch. serde_json added as a
dev-dependency only (test-only; never linked into the lib or the wheel, so
the crate stays runtime-dependency-free).
Independently proven at this commit: native output equals the committed
golden BIT-FOR-BIT (128/128 exact bits, Δ=0) for both base and loaded — the
golden is native-faithful today; this test keeps it that way.
2. MEDIUM — the Python parity helper passed non-finite output. `abs(nan - b) >
tol` is False, so an all-NaN embedding slipped through. Now every element
must be `math.isfinite` first. Proven: an all-NaN vector is rejected
("element 0 is not finite (nan)"); the Rust helpers already caught it via
`<=`.
3. LOW — a coherent shift inside the per-element tolerance could move the whole
vector undetected. Added a whole-vector cosine-similarity bound (≥ 1 - 1e-6)
alongside the per-element check.
Verified on aarch64/macOS: `test_aether` 13, full `python/tests/` suite 227
passed against a `--features sota` build; the new native test passes
(2/2) run standalone (the in-worktree `cargo test -p` only failed on an
un-checked-out submodule; ci.yml checks out submodules recursively).
Co-Authored-By: Ruflo & AQE
This commit is contained in:
parent
2febbb813c
commit
bc0e8fd031
|
|
@ -37,25 +37,43 @@ def load_input() -> list[list[float]]:
|
||||||
|
|
||||||
|
|
||||||
def assert_embedding_matches_golden(embedding: list[float], golden_name: str) -> None:
|
def assert_embedding_matches_golden(embedding: list[float], golden_name: str) -> None:
|
||||||
"""Assert `embedding` matches the committed golden vector within tolerance."""
|
"""Assert `embedding` matches the committed golden vector.
|
||||||
|
|
||||||
|
Two independent checks, because a per-element tolerance alone is not enough:
|
||||||
|
- **Per element**, within atol+rtol — catches a single component drifting.
|
||||||
|
- **Whole-vector cosine** ≥ 1 - 1e-6 — catches a *coherent* shift that stays
|
||||||
|
inside the per-element bound on every component yet moves the vector as a
|
||||||
|
whole (the failure mode a loose element tolerance would hide).
|
||||||
|
|
||||||
|
NaN/inf are rejected explicitly: `abs(nan - b) > tol` is False, so a bare
|
||||||
|
tolerance check would silently PASS an all-NaN embedding. Every value must be
|
||||||
|
finite first.
|
||||||
|
"""
|
||||||
golden = json.loads((GOLDEN / golden_name).read_text())
|
golden = json.loads((GOLDEN / golden_name).read_text())
|
||||||
assert len(embedding) == len(golden), (
|
assert len(embedding) == len(golden), (
|
||||||
f"{golden_name}: length {len(embedding)} != golden {len(golden)}"
|
f"{golden_name}: length {len(embedding)} != golden {len(golden)}"
|
||||||
)
|
)
|
||||||
worst = max(
|
for i, x in enumerate(embedding):
|
||||||
(abs(a - b), i, a, b)
|
assert math.isfinite(x), f"{golden_name}: element {i} is not finite ({x})"
|
||||||
for i, (a, b) in enumerate(zip(embedding, golden))
|
|
||||||
if abs(a - b) > PARITY_ATOL + PARITY_RTOL * abs(b)
|
for i, (a, b) in enumerate(zip(embedding, golden)):
|
||||||
) if any(
|
tol = PARITY_ATOL + PARITY_RTOL * abs(b)
|
||||||
abs(a - b) > PARITY_ATOL + PARITY_RTOL * abs(b)
|
assert abs(a - b) <= tol, (
|
||||||
for a, b in zip(embedding, golden)
|
f"{golden_name}: element {i} diverged from native golden beyond "
|
||||||
) else None
|
f"tolerance (got {a}, golden {b}, |Δ|={abs(a - b):.3e}) — "
|
||||||
assert worst is None, (
|
|
||||||
f"{golden_name}: element {worst[1]} diverged from native golden beyond "
|
|
||||||
f"tolerance (got {worst[2]}, golden {worst[3]}, |Δ|={worst[0]:.3e}) — "
|
|
||||||
"a real regression, not cross-arch f32 drift."
|
"a real regression, not cross-arch f32 drift."
|
||||||
)
|
)
|
||||||
|
|
||||||
|
dot = sum(a * b for a, b in zip(embedding, golden))
|
||||||
|
na = math.sqrt(sum(a * a for a in embedding))
|
||||||
|
nb = math.sqrt(sum(b * b for b in golden))
|
||||||
|
cosine = dot / (na * nb) if na > 0 and nb > 0 else 0.0
|
||||||
|
assert cosine >= 1.0 - 1e-6, (
|
||||||
|
f"{golden_name}: whole-vector cosine similarity to the golden is "
|
||||||
|
f"{cosine:.9f} (< 1 - 1e-6) — a coherent shift the per-element "
|
||||||
|
"tolerance did not catch."
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def build_extractor() -> aether.EmbeddingExtractor:
|
def build_extractor() -> aether.EmbeddingExtractor:
|
||||||
# Must match the native-Rust reference construction exactly.
|
# Must match the native-Rust reference construction exactly.
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,11 @@ categories.workspace = true
|
||||||
# ruvector server tree. Keep it std-only.
|
# ruvector server tree. Keep it std-only.
|
||||||
[dependencies]
|
[dependencies]
|
||||||
|
|
||||||
|
# Test-only: parses the committed golden fixtures shared with the Python parity
|
||||||
|
# tests. Never linked into the library or the wheel.
|
||||||
|
[dev-dependencies]
|
||||||
|
serde_json = "1"
|
||||||
|
|
||||||
[lib]
|
[lib]
|
||||||
name = "wifi_densepose_aether"
|
name = "wifi_densepose_aether"
|
||||||
path = "src/lib.rs"
|
path = "src/lib.rs"
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,115 @@
|
||||||
|
//! ADR-185 §4.1 — NATIVE half of the AETHER parity gate, and the half that runs
|
||||||
|
//! in CI.
|
||||||
|
//!
|
||||||
|
//! The committed golden vectors live under `python/tests/golden/` and are
|
||||||
|
//! shared with `python/tests/test_aether.py` (the binding half). That pytest
|
||||||
|
//! runs in python-ci; the native reference tests in `python/tests/*.rs` link
|
||||||
|
//! against the PyO3 crate and are NOT run by any workflow. This test closes that
|
||||||
|
//! gap: it recomputes the embedding through THIS std-only crate — no PyO3, no
|
||||||
|
//! marshalling — and asserts it matches the same golden within tolerance.
|
||||||
|
//!
|
||||||
|
//! Together with the pytest half: native≈golden AND binding≈golden ⇒
|
||||||
|
//! binding≈native, portably. And because this side has no marshalling, a
|
||||||
|
//! binding-specific defect baked into the golden would surface here as a native
|
||||||
|
//! mismatch — which is the failure mode a binding-derived golden + pytest-only
|
||||||
|
//! CI would otherwise hide.
|
||||||
|
//!
|
||||||
|
//! Runs under the repo's `cargo test --workspace` (this crate is a member).
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
use wifi_densepose_aether::embedding::{EmbeddingConfig, EmbeddingExtractor};
|
||||||
|
use wifi_densepose_aether::graph_transformer::TransformerConfig;
|
||||||
|
|
||||||
|
// Same tolerance as the Python and .rs parity tests. f32 + transcendentals are
|
||||||
|
// not bit-reproducible across arch, so the golden (generated on one machine) is
|
||||||
|
// compared within a bound, not by hash.
|
||||||
|
const PARITY_ATOL: f32 = 1e-4;
|
||||||
|
const PARITY_RTOL: f32 = 1e-4;
|
||||||
|
|
||||||
|
fn golden_dir() -> PathBuf {
|
||||||
|
// This crate lives at v2/crates/wifi-densepose-aether; the shared golden
|
||||||
|
// fixtures are the single source of truth under python/tests/golden.
|
||||||
|
PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.join("../../../python/tests/golden")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_vec(name: &str) -> Vec<f32> {
|
||||||
|
let raw = std::fs::read_to_string(golden_dir().join(name))
|
||||||
|
.unwrap_or_else(|e| panic!("read {name}: {e}"));
|
||||||
|
serde_json::from_str(&raw).unwrap_or_else(|e| panic!("parse {name}: {e}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn load_input() -> Vec<Vec<f32>> {
|
||||||
|
let raw = std::fs::read_to_string(golden_dir().join("aether_input.json"))
|
||||||
|
.expect("read aether_input.json");
|
||||||
|
let rows: Vec<Vec<f64>> = serde_json::from_str(&raw).expect("parse aether_input.json");
|
||||||
|
rows.into_iter()
|
||||||
|
.map(|r| r.into_iter().map(|x| x as f32).collect())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn extractor() -> EmbeddingExtractor {
|
||||||
|
let e = EmbeddingConfig { d_model: 64, d_proj: 128, temperature: 0.07, normalize: true };
|
||||||
|
let t = TransformerConfig {
|
||||||
|
n_subcarriers: 56,
|
||||||
|
n_keypoints: 17,
|
||||||
|
d_model: 64,
|
||||||
|
n_heads: 4,
|
||||||
|
n_gnn_layers: 2,
|
||||||
|
};
|
||||||
|
EmbeddingExtractor::new(t, e)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn formula_weights(n: usize) -> Vec<f32> {
|
||||||
|
(0..n)
|
||||||
|
.map(|i| ((i as u64 * 1103515245 + 12345) % 65536) as f32 / 65536.0 - 0.5)
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_matches_golden(embedding: &[f32], name: &str) {
|
||||||
|
let golden = read_vec(name);
|
||||||
|
assert_eq!(embedding.len(), golden.len(), "{name}: length mismatch");
|
||||||
|
for (i, (&a, &b)) in embedding.iter().zip(&golden).enumerate() {
|
||||||
|
assert!(a.is_finite(), "{name}: element {i} is not finite ({a})");
|
||||||
|
let tol = PARITY_ATOL + PARITY_RTOL * b.abs();
|
||||||
|
assert!(
|
||||||
|
(a - b).abs() <= tol,
|
||||||
|
"{name}: element {i} diverged beyond tolerance \
|
||||||
|
(got {a}, golden {b}, |Δ|={}) — real regression, not arch drift",
|
||||||
|
(a - b).abs()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn native_base_embedding_matches_committed_golden() {
|
||||||
|
let emb = extractor().extract(&load_input());
|
||||||
|
assert_matches_golden(&emb, "aether_embedding.json");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn native_loaded_embedding_matches_committed_golden() {
|
||||||
|
let input = load_input();
|
||||||
|
let mut ext = extractor();
|
||||||
|
let baseline = ext.extract(&input);
|
||||||
|
|
||||||
|
let weights = formula_weights(ext.param_count());
|
||||||
|
let mut buf = Vec::new();
|
||||||
|
buf.extend_from_slice(b"AETHERW1");
|
||||||
|
buf.extend_from_slice(&(weights.len() as u32).to_le_bytes());
|
||||||
|
for w in &weights {
|
||||||
|
buf.extend_from_slice(&w.to_le_bytes());
|
||||||
|
}
|
||||||
|
let wpath = std::env::temp_dir().join(format!("aether_golden_parity_{}.bin", std::process::id()));
|
||||||
|
std::fs::write(&wpath, &buf).expect("write weights");
|
||||||
|
ext.load_weights(&wpath).expect("load_weights");
|
||||||
|
let _ = std::fs::remove_file(&wpath);
|
||||||
|
|
||||||
|
let loaded = ext.extract(&input);
|
||||||
|
assert!(
|
||||||
|
baseline.iter().zip(&loaded).any(|(a, b)| (a - b).abs() > 1e-6),
|
||||||
|
"load_weights had no effect vs the random-init baseline"
|
||||||
|
);
|
||||||
|
assert_matches_golden(&loaded, "aether_loaded_embedding.json");
|
||||||
|
}
|
||||||
Loading…
Reference in New Issue