mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-07 20:32:12 +02:00
feat: filter and cap GraphRAG reranker input across full stack (#1021)
- Filter out RDF/RDFS/OWL schema predicates (rdfs:domain, owl:inverseOf, etc.) from hop traversal, keeping rdf:type for data signal - Skip edges where reranker-visible components are unlabeled IRIs, since the cross-encoder cannot meaningfully score raw URIs - Add max-reranker-input safety cap (default 350) to prevent overloading the reranker, applied after filtering for maximum useful candidates - Expose max-reranker-input as per-request parameter through schema, translator, REST API, socket client, CLI, and OpenAPI spec - Update tests - Update tech spec
This commit is contained in:
cybermaggedon
GitHub
parent
76c4763b9b
commit
68e816e65c
10 changed files with 198 additions and 43 deletions
|
|
@ -34,6 +34,22 @@ logger = logging.getLogger(__name__)
|
|||
|
||||
LABEL="http://www.w3.org/2000/01/rdf-schema#label"
|
||||
|
||||
RDF_NS = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
|
||||
RDFS_NS = "http://www.w3.org/2000/01/rdf-schema#"
|
||||
OWL_NS = "http://www.w3.org/2002/07/owl#"
|
||||
RDF_TYPE = RDF_NS + "type"
|
||||
SCHEMA_NAMESPACES = (RDF_NS, RDFS_NS, OWL_NS)
|
||||
|
||||
|
||||
def is_schema_predicate(predicate):
|
||||
"""Return True if the predicate is an RDF/RDFS/OWL schema predicate.
|
||||
|
||||
rdf:type is excluded from filtering as it carries useful data signal.
|
||||
"""
|
||||
if predicate == RDF_TYPE:
|
||||
return False
|
||||
return predicate.startswith(SCHEMA_NAMESPACES)
|
||||
|
||||
|
||||
def term_to_string(term):
|
||||
"""Extract string value from a Term object."""
|
||||
|
|
@ -120,7 +136,8 @@ class Query:
|
|||
def __init__(
|
||||
self, rag, collection, verbose,
|
||||
entity_limit=50, triple_limit=30, max_subgraph_size=1000,
|
||||
max_path_length=2, edge_limit=25, track_usage=None,
|
||||
max_path_length=2, edge_limit=25, max_reranker_input=350,
|
||||
track_usage=None,
|
||||
):
|
||||
self.rag = rag
|
||||
self.collection = collection
|
||||
|
|
@ -130,6 +147,7 @@ class Query:
|
|||
self.max_subgraph_size = max_subgraph_size
|
||||
self.max_path_length = max_path_length
|
||||
self.edge_limit = edge_limit
|
||||
self.max_reranker_input = max_reranker_input
|
||||
self.track_usage = track_usage
|
||||
|
||||
async def extract_concepts(self, query):
|
||||
|
|
@ -346,7 +364,7 @@ class Query:
|
|||
hop_directions = {}
|
||||
for triple, direction in triples:
|
||||
triple_tuple = (str(triple.s), str(triple.p), str(triple.o))
|
||||
if triple_tuple[1] == LABEL:
|
||||
if is_schema_predicate(triple_tuple[1]):
|
||||
continue
|
||||
if triple_tuple in seen_edges:
|
||||
continue
|
||||
|
|
@ -385,25 +403,50 @@ class Query:
|
|||
# The reranker text highlights the NEW information relative
|
||||
# to the traversal direction: arriving from S means p,o are
|
||||
# new; from O means s,p are new; from P means s,o are new.
|
||||
# Edges where the reranker-visible components are unlabeled
|
||||
# IRIs are skipped — the cross-encoder can't score them.
|
||||
def is_iri(val):
|
||||
return val.startswith(("http://", "https://", "urn:"))
|
||||
|
||||
filtered_triples = []
|
||||
labeled_hop = []
|
||||
documents = []
|
||||
for s, p, o in hop_triples:
|
||||
ls = label_map.get(s, s)
|
||||
lp = label_map.get(p, p)
|
||||
lo = label_map.get(o, o)
|
||||
labeled_hop.append((ls, lp, lo))
|
||||
|
||||
documents = []
|
||||
for i, (triple_tuple, (ls, lp, lo)) in enumerate(
|
||||
zip(hop_triples, labeled_hop)
|
||||
):
|
||||
direction = hop_directions[triple_tuple]
|
||||
direction = hop_directions[(s, p, o)]
|
||||
if direction == self.FROM_S:
|
||||
if is_iri(lp) or is_iri(lo):
|
||||
continue
|
||||
text = f"{lp} {lo}"
|
||||
elif direction == self.FROM_O:
|
||||
if is_iri(ls) or is_iri(lp):
|
||||
continue
|
||||
text = f"{ls} {lp}"
|
||||
else:
|
||||
if is_iri(ls) or is_iri(lo):
|
||||
continue
|
||||
text = f"{ls} {lo}"
|
||||
documents.append({"id": str(i), "text": text})
|
||||
|
||||
idx = len(filtered_triples)
|
||||
filtered_triples.append((s, p, o))
|
||||
labeled_hop.append((ls, lp, lo))
|
||||
documents.append({"id": str(idx), "text": text})
|
||||
|
||||
hop_triples = filtered_triples
|
||||
|
||||
# Cap the number of candidates sent to the reranker
|
||||
if len(hop_triples) > self.max_reranker_input:
|
||||
if self.verbose:
|
||||
logger.debug(
|
||||
f"Hop {hop + 1}: truncating {len(hop_triples)} "
|
||||
f"candidates to {self.max_reranker_input}"
|
||||
)
|
||||
hop_triples = hop_triples[:self.max_reranker_input]
|
||||
labeled_hop = labeled_hop[:self.max_reranker_input]
|
||||
documents = documents[:self.max_reranker_input]
|
||||
|
||||
queries = [
|
||||
{"id": str(i), "text": c}
|
||||
|
|
@ -588,7 +631,7 @@ class GraphRag:
|
|||
async def query(
|
||||
self, query, collection = "default",
|
||||
entity_limit = 50, triple_limit = 30, max_subgraph_size = 1000,
|
||||
max_path_length = 2, edge_limit = 25,
|
||||
max_path_length = 2, edge_limit = 25, max_reranker_input = 350,
|
||||
streaming = False,
|
||||
chunk_callback = None,
|
||||
explain_callback = None, save_answer_callback = None,
|
||||
|
|
@ -642,6 +685,7 @@ class GraphRag:
|
|||
max_subgraph_size = max_subgraph_size,
|
||||
max_path_length = max_path_length,
|
||||
edge_limit = edge_limit,
|
||||
max_reranker_input = max_reranker_input,
|
||||
track_usage = track_usage,
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ class Processor(FlowProcessor):
|
|||
max_subgraph_size = params.get("max_subgraph_size", 150)
|
||||
max_path_length = params.get("max_path_length", 2)
|
||||
edge_limit = params.get("edge_limit", 25)
|
||||
max_reranker_input = params.get("max_reranker_input", 350)
|
||||
|
||||
super(Processor, self).__init__(
|
||||
**params | {
|
||||
|
|
@ -44,6 +45,7 @@ class Processor(FlowProcessor):
|
|||
"max_subgraph_size": max_subgraph_size,
|
||||
"max_path_length": max_path_length,
|
||||
"edge_limit": edge_limit,
|
||||
"max_reranker_input": max_reranker_input,
|
||||
}
|
||||
)
|
||||
|
||||
|
|
@ -52,6 +54,7 @@ class Processor(FlowProcessor):
|
|||
self.default_max_subgraph_size = max_subgraph_size
|
||||
self.default_max_path_length = max_path_length
|
||||
self.default_edge_limit = edge_limit
|
||||
self.default_max_reranker_input = max_reranker_input
|
||||
|
||||
# Workspace isolation is enforced by the flow layer (flow.workspace).
|
||||
# Per-request caching (see GraphRag) keeps within-request state
|
||||
|
|
@ -197,6 +200,11 @@ class Processor(FlowProcessor):
|
|||
else:
|
||||
edge_limit = self.default_edge_limit
|
||||
|
||||
if v.max_reranker_input:
|
||||
max_reranker_input = v.max_reranker_input
|
||||
else:
|
||||
max_reranker_input = self.default_max_reranker_input
|
||||
|
||||
async def save_answer(doc_id, answer_text):
|
||||
await flow.librarian.save_document(
|
||||
doc_id=doc_id,
|
||||
|
|
@ -226,8 +234,8 @@ class Processor(FlowProcessor):
|
|||
entity_limit = entity_limit, triple_limit = triple_limit,
|
||||
max_subgraph_size = max_subgraph_size,
|
||||
max_path_length = max_path_length,
|
||||
|
||||
edge_limit = edge_limit,
|
||||
max_reranker_input = max_reranker_input,
|
||||
streaming = True,
|
||||
chunk_callback = send_chunk,
|
||||
explain_callback = send_explainability,
|
||||
|
|
@ -242,8 +250,8 @@ class Processor(FlowProcessor):
|
|||
entity_limit = entity_limit, triple_limit = triple_limit,
|
||||
max_subgraph_size = max_subgraph_size,
|
||||
max_path_length = max_path_length,
|
||||
|
||||
edge_limit = edge_limit,
|
||||
max_reranker_input = max_reranker_input,
|
||||
explain_callback = send_explainability,
|
||||
save_answer_callback = save_answer,
|
||||
parent_uri = v.parent_uri,
|
||||
|
|
@ -346,6 +354,13 @@ class Processor(FlowProcessor):
|
|||
help=f'Max edges selected per hop by cross-encoder (default: 25)'
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--max-reranker-input',
|
||||
type=int,
|
||||
default=350,
|
||||
help=f'Max candidate edges sent to the reranker per hop (default: 350)'
|
||||
)
|
||||
|
||||
# Note: Explainability triples are now stored in the request's collection
|
||||
# with the named graph urn:graph:retrieval (no separate collection needed)
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue