fix(noxa-68r.4): replace qdrant-client with plain reqwest REST calls

qdrant-client v1.x is gRPC-only — reqwest feature is snapshot downloads only.
Rewrite QdrantStore using direct HTTP calls against Qdrant REST API (port 6333):
- collection_exists: GET /collections/{name}
- create_collection: PUT /collections/{name} + PUT /collections/{name}/index
- upsert: PUT /collections/{name}/points?wait=true (256-batch)
- delete_by_url: POST /collections/{name}/points/delete?wait=true
- search: POST /collections/{name}/points/search
No new dependencies — reqwest already in workspace.
This commit is contained in:
Jacob Magar 2026-04-12 08:48:47 -04:00
parent 5217b99601
commit 2e4343905f
3 changed files with 180 additions and 117 deletions

View file

@ -33,8 +33,7 @@ async-trait = "0.1"
# HTTP client (plain reqwest — no primp patches needed for TEI/Qdrant) # HTTP client (plain reqwest — no primp patches needed for TEI/Qdrant)
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
# Vector store — REST transport only (reqwest feature, no protoc required) # No qdrant-client crate — REST calls via plain reqwest (no protoc/gRPC dependency)
qdrant-client = { version = "1", default-features = false, features = ["reqwest"] }
# Chunking # Chunking
text-splitter = { version = "0.25", features = ["markdown", "tokenizers"] } text-splitter = { version = "0.25", features = ["markdown", "tokenizers"] }

View file

@ -58,7 +58,7 @@ pub enum EmbedProviderConfig {
#[serde(tag = "type", rename_all = "snake_case")] #[serde(tag = "type", rename_all = "snake_case")]
pub enum VectorStoreConfig { pub enum VectorStoreConfig {
Qdrant { Qdrant {
/// gRPC URL — port 6334. qdrant-client v1.x uses gRPC (tonic), NOT REST. /// REST URL — port 6333 (e.g. http://127.0.0.1:53333 if port-mapped).
url: String, url: String,
collection: String, collection: String,
/// Optional API key. Override with NOXA_RAG_QDRANT_API_KEY env var. /// Optional API key. Override with NOXA_RAG_QDRANT_API_KEY env var.

View file

@ -1,92 +1,139 @@
use async_trait::async_trait; use async_trait::async_trait;
use serde::{Deserialize, Serialize};
use qdrant_client::Qdrant; use serde_json::json;
use qdrant_client::qdrant::{
Condition, CreateCollectionBuilder, CreateFieldIndexCollectionBuilder, DeletePointsBuilder,
Distance, FieldType, Filter, HnswConfigDiffBuilder, PointStruct, SearchPointsBuilder,
UpsertPointsBuilder, VectorParamsBuilder,
};
use qdrant_client::Payload;
use crate::error::RagError; use crate::error::RagError;
use crate::store::VectorStore; use crate::store::VectorStore;
use crate::types::{Point, SearchResult}; use crate::types::{Point, SearchResult};
// ── REST request/response shapes ─────────────────────────────────────────────
#[derive(Deserialize)]
struct QdrantStatus {
status: String, // "ok" on success
}
#[derive(Deserialize)]
struct CollectionInfoResponse {
result: Option<serde_json::Value>,
}
#[derive(Serialize)]
struct UpsertRequest {
points: Vec<QdrantPoint>,
}
#[derive(Serialize)]
struct QdrantPoint {
id: String, // UUID string
vector: Vec<f32>,
payload: std::collections::HashMap<String, serde_json::Value>,
}
#[derive(Serialize)]
struct DeleteByFilterRequest {
filter: serde_json::Value,
}
#[derive(Serialize)]
struct SearchRequest {
vector: Vec<f32>,
limit: usize,
with_payload: bool,
score_threshold: Option<f32>,
}
#[derive(Deserialize)]
struct SearchResponse {
result: Vec<SearchHit>,
}
#[derive(Deserialize)]
struct SearchHit {
score: f32,
payload: Option<std::collections::HashMap<String, serde_json::Value>>,
}
// ── QdrantStore ───────────────────────────────────────────────────────────────
pub struct QdrantStore { pub struct QdrantStore {
client: Qdrant, client: reqwest::Client,
base_url: String, // e.g. "http://127.0.0.1:53333"
collection: String, collection: String,
#[allow(dead_code)]
uuid_namespace: uuid::Uuid, uuid_namespace: uuid::Uuid,
} }
impl QdrantStore { impl QdrantStore {
/// Create a new QdrantStore.
///
/// `url` should be the gRPC endpoint, typically `http://localhost:6334`.
/// The crate uses gRPC transport via tonic — the `reqwest` feature only
/// enables snapshot downloads, not a REST transport.
pub fn new( pub fn new(
url: &str, url: &str,
collection: String, collection: String,
api_key: Option<String>, api_key: Option<String>,
uuid_namespace: uuid::Uuid, uuid_namespace: uuid::Uuid,
) -> Result<Self, RagError> { ) -> Result<Self, RagError> {
let mut builder = Qdrant::from_url(url); let mut headers = reqwest::header::HeaderMap::new();
if let Some(key) = api_key { if let Some(key) = api_key {
builder = builder.api_key(key); headers.insert(
"api-key",
key.parse().map_err(|_| RagError::Config("invalid Qdrant api-key".into()))?,
);
} }
let client = builder let client = reqwest::Client::builder()
.default_headers(headers)
.build() .build()
.map_err(|e| RagError::Store(format!("failed to build qdrant client: {e}")))?; .map_err(|e| RagError::Config(format!("failed to build HTTP client: {e}")))?;
Ok(Self { Ok(Self {
client, client,
base_url: url.trim_end_matches('/').to_string(),
collection, collection,
uuid_namespace, uuid_namespace,
}) })
} }
/// Check whether the collection already exists. /// GET /collections/{name} → true if 200, false if 404.
pub async fn collection_exists(&self) -> Result<bool, RagError> { pub async fn collection_exists(&self) -> Result<bool, RagError> {
self.client let url = format!("{}/collections/{}", self.base_url, self.collection);
.collection_exists(&self.collection) let resp = self.client.get(&url).send().await?;
.await match resp.status().as_u16() {
.map_err(|e| RagError::Store(format!("collection_exists failed: {e}"))) 200 => Ok(true),
404 => Ok(false),
s => Err(RagError::Store(format!("collection_exists: unexpected HTTP {s}"))),
}
} }
/// Create the collection with cosine distance, HNSW m=16/ef_construct=200, /// PUT /collections/{name} — create with Cosine/HNSW + payload indexes.
/// and payload indexes on `url` + `domain`.
pub async fn create_collection(&self, dims: usize) -> Result<(), RagError> { pub async fn create_collection(&self, dims: usize) -> Result<(), RagError> {
let hnsw = HnswConfigDiffBuilder::default() let url = format!("{}/collections/{}", self.base_url, self.collection);
.m(16) let body = json!({
.ef_construct(200) "vectors": {
.build(); "size": dims,
"distance": "Cosine",
"on_disk": true,
"hnsw_config": { "m": 16, "ef_construct": 200 }
},
"on_disk_payload": true
});
let vectors = VectorParamsBuilder::new(dims as u64, Distance::Cosine) let resp = self.client.put(&url).json(&body).send().await?;
.on_disk(true) if !resp.status().is_success() {
.hnsw_config(hnsw); let text = resp.text().await.unwrap_or_default();
return Err(RagError::Store(format!("create_collection failed: {text}")));
}
self.client // Payload indexes for fast URL/domain filtering.
.create_collection( for (field, schema_type) in [("url", "keyword"), ("domain", "keyword")] {
CreateCollectionBuilder::new(&self.collection) let idx_url = format!(
.vectors_config(vectors) "{}/collections/{}/index",
.on_disk_payload(true), self.base_url, self.collection
) );
.await let idx_body = json!({ "field_name": field, "field_schema": schema_type });
.map_err(|e| RagError::Store(format!("create_collection failed: {e}")))?; let r = self.client.put(&idx_url).json(&idx_body).send().await?;
if !r.status().is_success() {
// Payload indexes for fast filtering by url and domain. let text = r.text().await.unwrap_or_default();
for field in ["url", "domain"] { return Err(RagError::Store(format!(
self.client "create_field_index({field}) failed: {text}"
.create_field_index(CreateFieldIndexCollectionBuilder::new( )));
&self.collection, }
field,
FieldType::Keyword,
))
.await
.map_err(|e| {
RagError::Store(format!("create_field_index({field}) failed: {e}"))
})?;
} }
Ok(()) Ok(())
@ -95,86 +142,107 @@ impl QdrantStore {
#[async_trait] #[async_trait]
impl VectorStore for QdrantStore { impl VectorStore for QdrantStore {
/// Upsert points into the collection in batches of 256. /// PUT /collections/{name}/points?wait=true in batches of 256.
async fn upsert(&self, points: Vec<Point>) -> Result<(), RagError> { async fn upsert(&self, points: Vec<Point>) -> Result<(), RagError> {
let url = format!(
"{}/collections/{}/points?wait=true",
self.base_url, self.collection
);
for chunk in points.chunks(256) { for chunk in points.chunks(256) {
let qdrant_points: Vec<PointStruct> = chunk let qdrant_points: Vec<QdrantPoint> = chunk
.iter() .iter()
.map(|p| { .map(|p| {
let mut payload = Payload::new(); let mut payload = std::collections::HashMap::new();
payload.insert("text", p.payload.text.as_str()); payload.insert("text".into(), json!(p.payload.text));
payload.insert("url", p.payload.url.as_str()); payload.insert("url".into(), json!(p.payload.url));
payload.insert("domain", p.payload.domain.as_str()); payload.insert("domain".into(), json!(p.payload.domain));
payload.insert("chunk_index", p.payload.chunk_index as i64); payload.insert("chunk_index".into(), json!(p.payload.chunk_index));
payload.insert("total_chunks", p.payload.total_chunks as i64); payload.insert("total_chunks".into(), json!(p.payload.total_chunks));
payload.insert("token_estimate", p.payload.token_estimate as i64); payload.insert("token_estimate".into(), json!(p.payload.token_estimate));
QdrantPoint {
PointStruct::new( id: p.id.to_string(),
p.id.to_string(), // UUID as string PointId vector: p.vector.clone(),
p.vector.clone(),
payload, payload,
) }
}) })
.collect(); .collect();
self.client let resp = self
.upsert_points( .client
UpsertPointsBuilder::new(&self.collection, qdrant_points).wait(true), .put(&url)
) .json(&UpsertRequest { points: qdrant_points })
.await .send()
.map_err(|e| RagError::Store(format!("upsert_points failed: {e}")))?; .await?;
if !resp.status().is_success() {
let text = resp.text().await.unwrap_or_default();
return Err(RagError::Store(format!("upsert failed: {text}")));
}
} }
Ok(()) Ok(())
} }
/// Delete all points whose `url` payload field matches the normalized URL. /// POST /collections/{name}/points/delete?wait=true filtered by url payload.
async fn delete_by_url(&self, url: &str) -> Result<(), RagError> { async fn delete_by_url(&self, url: &str) -> Result<(), RagError> {
let normalized = normalize_url(url); let normalized = normalize_url(url);
let endpoint = format!(
"{}/collections/{}/points/delete?wait=true",
self.base_url, self.collection
);
let body = DeleteByFilterRequest {
filter: json!({
"must": [{ "key": "url", "match": { "value": normalized } }]
}),
};
self.client let resp = self.client.post(&endpoint).json(&body).send().await?;
.delete_points( if !resp.status().is_success() {
DeletePointsBuilder::new(&self.collection) let text = resp.text().await.unwrap_or_default();
.points(Filter::must([Condition::matches( return Err(RagError::Store(format!("delete_by_url failed: {text}")));
"url", }
normalized.clone(),
)]))
.wait(true),
)
.await
.map_err(|e| RagError::Store(format!("delete_points failed: {e}")))?;
Ok(()) Ok(())
} }
/// Search for the nearest `limit` vectors and return their payloads. /// POST /collections/{name}/points/search
async fn search(&self, vector: &[f32], limit: usize) -> Result<Vec<SearchResult>, RagError> { async fn search(&self, vector: &[f32], limit: usize) -> Result<Vec<SearchResult>, RagError> {
let response = self let url = format!(
.client "{}/collections/{}/points/search",
.search_points( self.base_url, self.collection
SearchPointsBuilder::new(&self.collection, vector.to_vec(), limit as u64) );
.with_payload(true), let body = SearchRequest {
) vector: vector.to_vec(),
.await limit,
.map_err(|e| RagError::Store(format!("search_points failed: {e}")))?; with_payload: true,
score_threshold: None,
};
let resp = self.client.post(&url).json(&body).send().await?;
if !resp.status().is_success() {
let text = resp.text().await.unwrap_or_default();
return Err(RagError::Store(format!("search failed: {text}")));
}
let response: SearchResponse = resp.json().await?;
let results = response let results = response
.result .result
.into_iter() .into_iter()
.filter_map(|hit| { .filter_map(|hit| {
let text = hit.get("text").as_str()?.to_string(); let payload = hit.payload?;
let url = hit.get("url").as_str()?.to_string(); let text = payload.get("text")?.as_str()?.to_string();
let chunk_index = hit.get("chunk_index").as_integer().unwrap_or(0) as usize; let url = payload.get("url")?.as_str()?.to_string();
let token_estimate = let chunk_index = payload
hit.get("token_estimate").as_integer().unwrap_or(0) as usize; .get("chunk_index")
.and_then(|v| v.as_u64())
Some(SearchResult { .unwrap_or(0) as usize;
text, let token_estimate = payload
url, .get("token_estimate")
score: hit.score, .and_then(|v| v.as_u64())
chunk_index, .unwrap_or(0) as usize;
token_estimate, Some(SearchResult { text, url, score: hit.score, chunk_index, token_estimate })
})
}) })
.collect(); .collect();
@ -186,13 +254,9 @@ impl VectorStore for QdrantStore {
} }
} }
/// Normalize a URL for consistent storage and lookup: /// Strip fragment, trailing path slash, lowercase scheme+host (url crate already does the latter).
/// - Strip fragment
/// - Strip trailing slash from path
/// - Scheme and host are already lowercased by the `url` crate
fn normalize_url(url: &str) -> String { fn normalize_url(url: &str) -> String {
use url::Url; let Ok(mut parsed) = url::Url::parse(url) else {
let Ok(mut parsed) = Url::parse(url) else {
return url.to_string(); return url.to_string();
}; };
parsed.set_fragment(None); parsed.set_fragment(None);