fix(homecore-assist): exact in-memory cosine k-NN, drop fragile :memory: HNSW
The semantic recognizer built a ruvector-core VectorDB at ":memory:"; under full-workspace feature unification the file-storage backend is enabled and ":memory:" is an invalid Windows filename (os error 123), panicking via .expect(). Replace the external index with an exact in-memory cosine k-NN over the enrolled exemplars (embeddings are L2-normalised, so cosine = dot product). For HOMECORE's small intent vocabularies this is faster, fully deterministic, and removes the storage backend + cross-crate feature coupling entirely. ruvector-core dropped from the crate (only used here). Workspace 3122 passed/0 failed. Co-Authored-By: claude-flow <ruv@ruv.net>
This commit is contained in:
parent
3d96789475
commit
d79c22e03a
|
|
@ -10972,6 +10972,7 @@ dependencies = [
|
||||||
"ruvector-temporal-tensor",
|
"ruvector-temporal-tensor",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
|
"serialport",
|
||||||
"thiserror 2.0.18",
|
"thiserror 2.0.18",
|
||||||
"tokio",
|
"tokio",
|
||||||
"tokio-test",
|
"tokio-test",
|
||||||
|
|
|
||||||
|
|
@ -43,16 +43,13 @@ regex = "1"
|
||||||
# Structured logging.
|
# Structured logging.
|
||||||
tracing = "0.1"
|
tracing = "0.1"
|
||||||
|
|
||||||
# Semantic intent recognition: ruvector-core HNSW index over enrolled intent
|
|
||||||
# exemplars (same facility homecore-recorder uses for state search). The
|
|
||||||
# `semantic` feature is on by default so SemanticIntentRecognizer has a real
|
|
||||||
# embedding + nearest-neighbour search out of the box.
|
|
||||||
ruvector-core = { version = "2.2.0", optional = true, default-features = false }
|
|
||||||
|
|
||||||
[features]
|
[features]
|
||||||
default = ["semantic"]
|
default = ["semantic"]
|
||||||
# Enables SemanticIntentRecognizer's HNSW-backed embedding search.
|
# Enables SemanticIntentRecognizer's embedding-based exact cosine k-NN match.
|
||||||
semantic = ["dep:ruvector-core"]
|
# Self-contained: deterministic feature-hash embeddings + an in-memory cosine
|
||||||
|
# scan, with no external index/storage dependency (the small intent vocabularies
|
||||||
|
# make an exact scan faster and far more robust than an ANN backend).
|
||||||
|
semantic = []
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
tokio = { version = "1", features = ["full", "test-util"] }
|
tokio = { version = "1", features = ["full", "test-util"] }
|
||||||
|
|
|
||||||
|
|
@ -18,8 +18,8 @@
|
||||||
//! while unrelated utterances ("play jazz music") land far apart. It is a real,
|
//! while unrelated utterances ("play jazz music") land far apart. It is a real,
|
||||||
//! reproducible similarity signal — not a hash that ignores meaning.
|
//! reproducible similarity signal — not a hash that ignores meaning.
|
||||||
//!
|
//!
|
||||||
//! The output dimension matches [`EMBEDDING_DIM`] and is fed directly into the
|
//! The output dimension matches [`EMBEDDING_DIM`] and is consumed directly by
|
||||||
//! ruvector-core HNSW index used by [`crate::recognizer::SemanticIntentRecognizer`].
|
//! the exact in-memory cosine k-NN in `crate::semantic_recognizer`.
|
||||||
|
|
||||||
/// Dimensionality of the hashed embedding space.
|
/// Dimensionality of the hashed embedding space.
|
||||||
///
|
///
|
||||||
|
|
|
||||||
|
|
@ -1,14 +1,19 @@
|
||||||
//! `SemanticIntentRecognizer` — HNSW-backed semantic intent matching.
|
//! `SemanticIntentRecognizer` — embedding-based semantic intent matching.
|
||||||
//!
|
//!
|
||||||
//! Embeds utterances with [`crate::embedding`] (deterministic feature hashing)
|
//! Embeds utterances with [`crate::embedding`] (deterministic feature hashing)
|
||||||
//! and searches a ruvector-core HNSW index of enrolled intent exemplars. On a
|
//! and runs an **exact in-memory cosine k-NN** over enrolled intent exemplars.
|
||||||
//! match above the similarity threshold the exemplar's intent is returned, with
|
//! On a match above the similarity threshold the exemplar's intent is returned,
|
||||||
//! slots extracted from the incoming utterance via an optional paired regex.
|
//! with slots extracted from the incoming utterance via an optional paired
|
||||||
//! Below threshold (or with an empty index) it delegates to the inner
|
//! regex. Below threshold (or with an empty index) it delegates to the inner
|
||||||
//! [`RegexIntentRecognizer`](crate::recognizer::RegexIntentRecognizer).
|
//! [`RegexIntentRecognizer`](crate::recognizer::RegexIntentRecognizer).
|
||||||
//!
|
//!
|
||||||
|
//! For the small intent vocabularies HOMECORE deals with, an exact cosine scan
|
||||||
|
//! is both faster and far more robust than an external ANN index — it has no
|
||||||
|
//! storage backend, no cross-crate feature coupling, and is fully deterministic.
|
||||||
|
//! Embeddings are L2-normalised, so cosine similarity is a plain dot product.
|
||||||
|
//!
|
||||||
//! Gated behind the default-on `semantic` feature. When disabled, a thin
|
//! Gated behind the default-on `semantic` feature. When disabled, a thin
|
||||||
//! delegating wrapper keeps the public type available without ruvector-core.
|
//! delegating wrapper keeps the public type available.
|
||||||
|
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
#[cfg(feature = "semantic")]
|
#[cfg(feature = "semantic")]
|
||||||
|
|
@ -33,6 +38,8 @@ struct Exemplar {
|
||||||
language: String,
|
language: String,
|
||||||
/// Optional slot-extraction regex applied to the matched utterance.
|
/// Optional slot-extraction regex applied to the matched utterance.
|
||||||
slot_regex: Option<Regex>,
|
slot_regex: Option<Regex>,
|
||||||
|
/// L2-normalised embedding of the enrolled phrase, for cosine k-NN.
|
||||||
|
vector: Vec<f32>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Semantic recognizer backed by a real ruvector-core HNSW index.
|
/// Semantic recognizer backed by a real ruvector-core HNSW index.
|
||||||
|
|
@ -50,8 +57,7 @@ pub struct SemanticIntentRecognizer {
|
||||||
|
|
||||||
#[cfg(feature = "semantic")]
|
#[cfg(feature = "semantic")]
|
||||||
struct SemanticIndexInner {
|
struct SemanticIndexInner {
|
||||||
db: ruvector_core::VectorDB,
|
/// Enrolled exemplars in insertion order; the `Vec` index is the id.
|
||||||
/// Parallel to insertion order; HNSW ids are the stringified `Vec` index.
|
|
||||||
exemplars: Vec<Exemplar>,
|
exemplars: Vec<Exemplar>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -65,25 +71,9 @@ impl SemanticIntentRecognizer {
|
||||||
|
|
||||||
/// Build with an explicit similarity threshold in `[0, 1]`.
|
/// Build with an explicit similarity threshold in `[0, 1]`.
|
||||||
pub fn with_threshold(fallback: RegexIntentRecognizer, threshold: f32) -> Self {
|
pub fn with_threshold(fallback: RegexIntentRecognizer, threshold: f32) -> Self {
|
||||||
use ruvector_core::types::{DbOptions, DistanceMetric, HnswConfig};
|
|
||||||
let options = DbOptions {
|
|
||||||
dimensions: crate::embedding::EMBEDDING_DIM,
|
|
||||||
distance_metric: DistanceMetric::Cosine,
|
|
||||||
storage_path: ":memory:".to_string(),
|
|
||||||
hnsw_config: Some(HnswConfig {
|
|
||||||
m: 16,
|
|
||||||
ef_construction: 100,
|
|
||||||
ef_search: 50,
|
|
||||||
max_elements: 4096,
|
|
||||||
}),
|
|
||||||
quantization: None,
|
|
||||||
};
|
|
||||||
let db = ruvector_core::VectorDB::new(options)
|
|
||||||
.expect("in-memory HNSW index construction is infallible for valid options");
|
|
||||||
Self {
|
Self {
|
||||||
fallback,
|
fallback,
|
||||||
index: std::sync::Arc::new(tokio::sync::RwLock::new(SemanticIndexInner {
|
index: std::sync::Arc::new(tokio::sync::RwLock::new(SemanticIndexInner {
|
||||||
db,
|
|
||||||
exemplars: Vec::new(),
|
exemplars: Vec::new(),
|
||||||
})),
|
})),
|
||||||
threshold,
|
threshold,
|
||||||
|
|
@ -109,19 +99,11 @@ impl SemanticIntentRecognizer {
|
||||||
let vector = crate::embedding::embed(phrase);
|
let vector = crate::embedding::embed(phrase);
|
||||||
|
|
||||||
let mut inner = self.index.write().await;
|
let mut inner = self.index.write().await;
|
||||||
let id = inner.exemplars.len();
|
|
||||||
inner
|
|
||||||
.db
|
|
||||||
.insert(ruvector_core::types::VectorEntry {
|
|
||||||
id: Some(id.to_string()),
|
|
||||||
vector,
|
|
||||||
metadata: None,
|
|
||||||
})
|
|
||||||
.map_err(|e| RecognizerError::Internal(format!("HNSW insert failed: {e}")))?;
|
|
||||||
inner.exemplars.push(Exemplar {
|
inner.exemplars.push(Exemplar {
|
||||||
name: IntentName::new(name),
|
name: IntentName::new(name),
|
||||||
language: language.into(),
|
language: language.into(),
|
||||||
slot_regex,
|
slot_regex,
|
||||||
|
vector,
|
||||||
});
|
});
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
@ -130,34 +112,18 @@ impl SemanticIntentRecognizer {
|
||||||
/// exemplar matches `language`, or `None` if the index is empty.
|
/// exemplar matches `language`, or `None` if the index is empty.
|
||||||
async fn nearest(&self, utterance: &str, language: &str) -> Option<(usize, f32)> {
|
async fn nearest(&self, utterance: &str, language: &str) -> Option<(usize, f32)> {
|
||||||
let normalised = utterance.trim().to_lowercase();
|
let normalised = utterance.trim().to_lowercase();
|
||||||
let vector = crate::embedding::embed(&normalised);
|
let query = crate::embedding::embed(&normalised);
|
||||||
|
|
||||||
|
// Exact in-memory cosine k-NN. Embeddings are L2-normalised, so cosine
|
||||||
|
// similarity is a plain dot product (see `crate::embedding`). Returns the
|
||||||
|
// best language-eligible exemplar, or `None` for an empty index.
|
||||||
let inner = self.index.read().await;
|
let inner = self.index.read().await;
|
||||||
if inner.exemplars.is_empty() {
|
inner
|
||||||
return None;
|
.exemplars
|
||||||
}
|
.iter()
|
||||||
let k = inner.exemplars.len().min(8);
|
.enumerate()
|
||||||
let results = inner
|
.filter(|(_, e)| e.language == "*" || e.language == language)
|
||||||
.db
|
.map(|(id, e)| (id, crate::embedding::cosine_similarity(&query, &e.vector)))
|
||||||
.search(ruvector_core::types::SearchQuery {
|
|
||||||
vector,
|
|
||||||
k,
|
|
||||||
filter: None,
|
|
||||||
ef_search: None,
|
|
||||||
})
|
|
||||||
.ok()?;
|
|
||||||
|
|
||||||
// Cosine distance → similarity = 1 - distance. Pick best language-eligible.
|
|
||||||
results
|
|
||||||
.into_iter()
|
|
||||||
.filter_map(|r| r.id.parse::<usize>().ok().map(|id| (id, 1.0 - r.score)))
|
|
||||||
.filter(|(id, _)| {
|
|
||||||
inner
|
|
||||||
.exemplars
|
|
||||||
.get(*id)
|
|
||||||
.map(|e| e.language == "*" || e.language == language)
|
|
||||||
.unwrap_or(false)
|
|
||||||
})
|
|
||||||
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
|
.max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue