mirror of
https://github.com/ModernRelay/omnigraph.git
synced 2026-07-12 03:12:11 +02:00
fix(engine): reject non-finite and zero-norm embeddings
Validate every component is_finite() (naming the offending index) and
reject zero-norm vectors before normalizing, via the existing
Err(String) channel — callers already map it through EmbedCallError
{retryable:false} into OmniError::manifest_internal, so no new error
plumbing. The red tests turn green; docs state the finite/non-zero
contract in embeddings.md.
Closes iss-embedding-nan-validation.
This commit is contained in:
parent
3b00d81626
commit
10835ca788
2 changed files with 23 additions and 1 deletions
|
|
@ -542,6 +542,25 @@ fn validate_and_normalize_embedding(
|
|||
values.len()
|
||||
));
|
||||
}
|
||||
// Reject poisoned vectors BEFORE normalizing: a NaN component makes the
|
||||
// norm NaN (whose `> EPSILON` comparison is false, silently skipping
|
||||
// normalization), an Inf component normalizes the rest to 0s and itself
|
||||
// to NaN, and a zero vector has no direction — all three would persist
|
||||
// as "successful" embeddings and degrade ANN/IVF for unrelated rows.
|
||||
if let Some(idx) = values.iter().position(|v| !v.is_finite()) {
|
||||
return Err(format!(
|
||||
"embedding component {} is not finite ({})",
|
||||
idx, values[idx]
|
||||
));
|
||||
}
|
||||
let norm = values
|
||||
.iter()
|
||||
.map(|v| (*v as f64) * (*v as f64))
|
||||
.sum::<f64>()
|
||||
.sqrt() as f32;
|
||||
if norm <= f32::EPSILON {
|
||||
return Err("embedding has zero norm (no direction)".to_string());
|
||||
}
|
||||
Ok(normalize_vector(values))
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -13,7 +13,10 @@ query vectors and document vectors share one model and one vector space.
|
|||
| `gemini` | `POST {base}/models/{model}:embedContent`, `x-goog-api-key`, with `RETRIEVAL_QUERY` / `RETRIEVAL_DOCUMENT` task types | Reaching Google's `generativelanguage` API directly |
|
||||
| `mock` | none — deterministic offline vectors | Tests and local dev without a key |
|
||||
|
||||
Vectors are stored L2-normalized as `FixedSizeList(Float32, dim)`; the requested output dimension is driven by
|
||||
Vectors are stored L2-normalized as `FixedSizeList(Float32, dim)`. Embedding
|
||||
values must be finite — a vector containing NaN/Inf, or an all-zero vector
|
||||
(no direction under cosine distance), is rejected at the client boundary
|
||||
rather than persisted; the requested output dimension is driven by
|
||||
the target column width and sent as Gemini `outputDimensionality` / OpenAI `dimensions`.
|
||||
|
||||
## Configuration (cluster)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue