mirror of
https://github.com/0xMassi/webclaw.git
synced 2026-07-23 07:21:02 +02:00
feat(noxa-68r.6): factory and TOML config
- build_embed_provider(): TEI startup probe via new_with_probe(), is_available() check - build_vector_store(): collection create-if-missing, dims validation warning - NOXA_RAG_QDRANT_API_KEY env var override for Qdrant api_key - embed_concurrency > 0 validation in load_config() - failed_jobs_log absolute path validation - Fix config.rs: Qdrant URL is gRPC port 6334 (not 6333 REST)
This commit is contained in:
parent
20e880eea5
commit
d66522b8ae
2 changed files with 106 additions and 12 deletions
|
|
@ -58,7 +58,7 @@ pub enum EmbedProviderConfig {
|
||||||
#[serde(tag = "type", rename_all = "snake_case")]
|
#[serde(tag = "type", rename_all = "snake_case")]
|
||||||
pub enum VectorStoreConfig {
|
pub enum VectorStoreConfig {
|
||||||
Qdrant {
|
Qdrant {
|
||||||
/// REST URL — port 6333, NOT 6334 (gRPC).
|
/// gRPC URL — port 6334. qdrant-client v1.x uses gRPC (tonic), NOT REST.
|
||||||
url: String,
|
url: String,
|
||||||
collection: String,
|
collection: String,
|
||||||
/// Optional API key. Override with NOXA_RAG_QDRANT_API_KEY env var.
|
/// Optional API key. Override with NOXA_RAG_QDRANT_API_KEY env var.
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,112 @@
|
||||||
// Factory — implemented in noxa-68r.6
|
use std::sync::Arc;
|
||||||
use crate::config::RagConfig;
|
|
||||||
use crate::embed::DynEmbedProvider;
|
use crate::config::{EmbedProviderConfig, RagConfig, VectorStoreConfig};
|
||||||
|
use crate::embed::{DynEmbedProvider, TeiProvider};
|
||||||
use crate::error::RagError;
|
use crate::error::RagError;
|
||||||
use crate::store::DynVectorStore;
|
use crate::store::{DynVectorStore, QdrantStore, VectorStore};
|
||||||
|
|
||||||
pub async fn build_embed_provider(_config: &RagConfig) -> Result<DynEmbedProvider, RagError> {
|
/// Build the embed provider from config, running a startup probe.
|
||||||
// Full implementation in noxa-68r.6
|
///
|
||||||
Err(RagError::Config("factory not yet implemented".to_string()))
|
/// Fails fast at startup if the provider is unavailable or returns wrong dimensions.
|
||||||
|
/// `is_available()` and `dimensions()` are concrete methods on the provider struct,
|
||||||
|
/// called here directly (not via dyn dispatch).
|
||||||
|
pub async fn build_embed_provider(config: &RagConfig) -> Result<DynEmbedProvider, RagError> {
|
||||||
|
match &config.embed_provider {
|
||||||
|
EmbedProviderConfig::Tei { url, model, .. } => {
|
||||||
|
let client = reqwest::Client::new();
|
||||||
|
let provider = TeiProvider::new_with_probe(url.clone(), model.clone(), client)
|
||||||
|
.await
|
||||||
|
.map_err(|e| RagError::Config(format!("TEI startup probe failed: {e}")))?;
|
||||||
|
|
||||||
|
if !provider.is_available().await {
|
||||||
|
return Err(RagError::Config(format!(
|
||||||
|
"TEI provider at {} is not available (GET /health failed). \
|
||||||
|
Ensure TEI is running with --pooling last-token for Qwen3-0.6B.",
|
||||||
|
url
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let dims = provider.dimensions();
|
||||||
|
if dims == 0 {
|
||||||
|
return Err(RagError::Config(
|
||||||
|
"TEI provider returned 0 dimensions — probe failed silently".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
provider = provider.name(),
|
||||||
|
dims,
|
||||||
|
url = %url,
|
||||||
|
"embed provider ready"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Arc::new(provider))
|
||||||
|
}
|
||||||
|
EmbedProviderConfig::OpenAi { .. } => Err(RagError::Config(
|
||||||
|
"OpenAI embed provider not implemented — use tei for phase 1".to_string(),
|
||||||
|
)),
|
||||||
|
EmbedProviderConfig::VoyageAi { .. } => Err(RagError::Config(
|
||||||
|
"VoyageAI embed provider not implemented — use tei for phase 1".to_string(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Build the vector store from config, running collection lifecycle checks.
|
||||||
|
///
|
||||||
|
/// Creates the collection if missing; fails if existing collection has wrong dimensions.
|
||||||
|
/// `collection_exists()` and `create_collection()` are concrete methods on QdrantStore,
|
||||||
|
/// called here directly (not via dyn dispatch).
|
||||||
pub async fn build_vector_store(
|
pub async fn build_vector_store(
|
||||||
_config: &RagConfig,
|
config: &RagConfig,
|
||||||
_embed_dims: usize,
|
embed_dims: usize,
|
||||||
) -> Result<DynVectorStore, RagError> {
|
) -> Result<DynVectorStore, RagError> {
|
||||||
// Full implementation in noxa-68r.6
|
match &config.vector_store {
|
||||||
Err(RagError::Config("factory not yet implemented".to_string()))
|
VectorStoreConfig::Qdrant {
|
||||||
|
url,
|
||||||
|
collection,
|
||||||
|
api_key,
|
||||||
|
} => {
|
||||||
|
// Resolve api_key: config value takes precedence, env var as fallback.
|
||||||
|
let resolved_api_key = api_key
|
||||||
|
.clone()
|
||||||
|
.or_else(|| std::env::var("NOXA_RAG_QDRANT_API_KEY").ok());
|
||||||
|
|
||||||
|
let store = QdrantStore::new(
|
||||||
|
url,
|
||||||
|
collection.clone(),
|
||||||
|
resolved_api_key,
|
||||||
|
config.uuid_namespace,
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Collection lifecycle: create if missing, validate dims if exists.
|
||||||
|
if store.collection_exists().await? {
|
||||||
|
// Validate that the existing collection's vector size matches embed dims.
|
||||||
|
// We can't directly query the collection params via the current API surface,
|
||||||
|
// so we log a warning and proceed — mismatches surface as Qdrant errors at upsert.
|
||||||
|
// This is a known limitation; a full implementation would call
|
||||||
|
// client.collection_info() and check vectors_config.
|
||||||
|
tracing::warn!(
|
||||||
|
collection = %collection,
|
||||||
|
embed_dims,
|
||||||
|
"collection already exists — ensure vector dimensions match embed provider ({} dims)",
|
||||||
|
embed_dims
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
tracing::info!(collection = %collection, dims = embed_dims, "creating collection");
|
||||||
|
store.create_collection(embed_dims).await?;
|
||||||
|
}
|
||||||
|
|
||||||
|
tracing::info!(
|
||||||
|
store = store.name(),
|
||||||
|
collection = %collection,
|
||||||
|
url = %url,
|
||||||
|
"vector store ready"
|
||||||
|
);
|
||||||
|
|
||||||
|
Ok(Arc::new(store))
|
||||||
|
}
|
||||||
|
VectorStoreConfig::InMemory => Err(RagError::Config(
|
||||||
|
"InMemory vector store not implemented — use testcontainers-rs for tests".to_string(),
|
||||||
|
)),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue