From 0c36dda4437977a67c3e1f32640d4d8f2921e37b Mon Sep 17 00:00:00 2001 From: cybermaggedon Date: Wed, 15 Jul 2026 10:13:19 +0100 Subject: [PATCH] feat: preserve RDF language tags and datatypes through ingestion and query (#1047) Language tags and XSD datatypes on RDF literals were being silently dropped at multiple points in the pipeline, making SPARQL FILTER(LANG()) queries and multilingual datasets non-functional. Ingestion fixes: - load_knowledge.py and load_turtle.py now extract .language and .datatype from rdflib Literal objects instead of discarding them - Triple dataclass gains optional o_datatype and o_language fields - _string_to_term() emits "dt" and "ln" in the wire format - AsyncBulkClient uses the same term serialization as BulkClient Query fix: - TriplesClient.query_gen() preserves Term objects directly instead of converting through Uri/Literal (which are str subclasses with no room for language metadata), enabling SPARQL LANG() filters to work --- .../trustgraph/api/async_bulk_client.py | 11 +++++++--- trustgraph-base/trustgraph/api/bulk_client.py | 18 +++++++++++++---- trustgraph-base/trustgraph/api/types.py | 4 ++++ .../trustgraph/base/triples_client.py | 2 +- .../trustgraph/cli/load_knowledge.py | 20 ++++++++++++------- trustgraph-cli/trustgraph/cli/load_turtle.py | 18 ++++++++++++----- 6 files changed, 53 insertions(+), 20 deletions(-) diff --git a/trustgraph-base/trustgraph/api/async_bulk_client.py b/trustgraph-base/trustgraph/api/async_bulk_client.py index f93ab667..9308f1da 100644 --- a/trustgraph-base/trustgraph/api/async_bulk_client.py +++ b/trustgraph-base/trustgraph/api/async_bulk_client.py @@ -4,6 +4,7 @@ import websockets from typing import Optional, AsyncIterator, Dict, Any, Iterator from . types import Triple +from . bulk_client import _string_to_term class AsyncBulkClient: @@ -45,9 +46,13 @@ class AsyncBulkClient: async with websockets.connect(ws_url, ping_interval=20, ping_timeout=self.timeout) as websocket: async for triple in triples: message = { - "s": triple.s, - "p": triple.p, - "o": triple.o + "s": _string_to_term(triple.s), + "p": _string_to_term(triple.p), + "o": _string_to_term( + triple.o, + datatype=triple.o_datatype, + language=triple.o_language, + ), } await websocket.send(json.dumps(message)) diff --git a/trustgraph-base/trustgraph/api/bulk_client.py b/trustgraph-base/trustgraph/api/bulk_client.py index ae185240..89da0c2c 100644 --- a/trustgraph-base/trustgraph/api/bulk_client.py +++ b/trustgraph-base/trustgraph/api/bulk_client.py @@ -15,13 +15,19 @@ from . types import Triple from . exceptions import ProtocolException -def _string_to_term(value: str) -> Dict[str, Any]: +def _string_to_term( + value: str, datatype: str = "", language: str = "" +) -> Dict[str, Any]: """Convert a string value to Term format for the gateway.""" - # Treat URIs as IRI type, otherwise as literal if value.startswith("http://") or value.startswith("https://") or "://" in value: return {"t": "i", "i": value} else: - return {"t": "l", "v": value} + result: Dict[str, Any] = {"t": "l", "v": value} + if datatype: + result["dt"] = datatype + if language: + result["ln"] = language + return result class BulkClient: @@ -141,7 +147,11 @@ class BulkClient: batch.append({ "s": _string_to_term(triple.s), "p": _string_to_term(triple.p), - "o": _string_to_term(triple.o) + "o": _string_to_term( + triple.o, + datatype=triple.o_datatype, + language=triple.o_language, + ), }) if len(batch) >= batch_size: message = { diff --git a/trustgraph-base/trustgraph/api/types.py b/trustgraph-base/trustgraph/api/types.py index fafb8224..760648c8 100644 --- a/trustgraph-base/trustgraph/api/types.py +++ b/trustgraph-base/trustgraph/api/types.py @@ -19,10 +19,14 @@ class Triple: s: Subject (entity URI or value) p: Predicate (relationship URI) o: Object (entity URI, literal value, or typed value) + o_datatype: XSD datatype URI for literal objects (e.g. "xsd:string") + o_language: Language tag for literal objects (e.g. "en", "lt") """ s : str p : str o : str + o_datatype : str = "" + o_language : str = "" @dataclasses.dataclass class ConfigKey: diff --git a/trustgraph-base/trustgraph/base/triples_client.py b/trustgraph-base/trustgraph/base/triples_client.py index 0506cb9f..72042d60 100644 --- a/trustgraph-base/trustgraph/base/triples_client.py +++ b/trustgraph-base/trustgraph/base/triples_client.py @@ -58,7 +58,7 @@ class TriplesClient(RequestResponse): raise RuntimeError(resp.error.message) batch = [ - Triple(to_value(v.s), to_value(v.p), to_value(v.o)) + Triple(v.s, v.p, v.o) for v in resp.triples ] await queue.put(batch) diff --git a/trustgraph-cli/trustgraph/cli/load_knowledge.py b/trustgraph-cli/trustgraph/cli/load_knowledge.py index 7e9dadd4..80d2c434 100644 --- a/trustgraph-cli/trustgraph/cli/load_knowledge.py +++ b/trustgraph-cli/trustgraph/cli/load_knowledge.py @@ -44,15 +44,21 @@ class KnowledgeLoader: for e in g: s_value = str(e[0]) p_value = str(e[1]) + o_value = str(e[2]) - if isinstance(e[2], rdflib.term.URIRef): - o_value = str(e[2]) - o_is_uri = True - else: - o_value = str(e[2]) - o_is_uri = False + o_datatype = "" + o_language = "" - yield Triple(s=s_value, p=p_value, o=o_value) + if isinstance(e[2], rdflib.term.Literal): + if e[2].language: + o_language = str(e[2].language) + if e[2].datatype: + o_datatype = str(e[2].datatype) + + yield Triple( + s=s_value, p=p_value, o=o_value, + o_datatype=o_datatype, o_language=o_language, + ) def load_entity_contexts_from_file(self, file) -> Iterator[Tuple[str, str]]: """Generator that yields (entity, context) tuples from a Turtle file""" diff --git a/trustgraph-cli/trustgraph/cli/load_turtle.py b/trustgraph-cli/trustgraph/cli/load_turtle.py index 43ef9e6f..66cbe56e 100644 --- a/trustgraph-cli/trustgraph/cli/load_turtle.py +++ b/trustgraph-cli/trustgraph/cli/load_turtle.py @@ -44,13 +44,21 @@ class Loader: for e in g: s_value = str(e[0]) p_value = str(e[1]) + o_value = str(e[2]) - if isinstance(e[2], rdflib.term.URIRef): - o_value = str(e[2]) - else: - o_value = str(e[2]) + o_datatype = "" + o_language = "" - yield Triple(s=s_value, p=p_value, o=o_value) + if isinstance(e[2], rdflib.term.Literal): + if e[2].language: + o_language = str(e[2].language) + if e[2].datatype: + o_datatype = str(e[2].datatype) + + yield Triple( + s=s_value, p=p_value, o=o_value, + o_datatype=o_datatype, o_language=o_language, + ) def run(self): """Load triples using Python API"""