feat: preserve RDF language tags and datatypes through ingestion and query (#1047)

Language tags and XSD datatypes on RDF literals were being silently
dropped at multiple points in the pipeline, making SPARQL FILTER(LANG())
queries and multilingual datasets non-functional.

Ingestion fixes:
- load_knowledge.py and load_turtle.py now extract .language and
  .datatype from rdflib Literal objects instead of discarding them
- Triple dataclass gains optional o_datatype and o_language fields
- _string_to_term() emits "dt" and "ln" in the wire format
- AsyncBulkClient uses the same term serialization as BulkClient

Query fix:
- TriplesClient.query_gen() preserves Term objects directly instead of
  converting through Uri/Literal (which are str subclasses with no room
  for language metadata), enabling SPARQL LANG() filters to work
This commit is contained in:
cybermaggedon 2026-07-15 10:13:19 +01:00 committed by GitHub
parent f9f5a2318f
commit 0c36dda443
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 53 additions and 20 deletions

View file

@ -44,15 +44,21 @@ class KnowledgeLoader:
for e in g:
s_value = str(e[0])
p_value = str(e[1])
o_value = str(e[2])
if isinstance(e[2], rdflib.term.URIRef):
o_value = str(e[2])
o_is_uri = True
else:
o_value = str(e[2])
o_is_uri = False
o_datatype = ""
o_language = ""
yield Triple(s=s_value, p=p_value, o=o_value)
if isinstance(e[2], rdflib.term.Literal):
if e[2].language:
o_language = str(e[2].language)
if e[2].datatype:
o_datatype = str(e[2].datatype)
yield Triple(
s=s_value, p=p_value, o=o_value,
o_datatype=o_datatype, o_language=o_language,
)
def load_entity_contexts_from_file(self, file) -> Iterator[Tuple[str, str]]:
"""Generator that yields (entity, context) tuples from a Turtle file"""

View file

@ -44,13 +44,21 @@ class Loader:
for e in g:
s_value = str(e[0])
p_value = str(e[1])
o_value = str(e[2])
if isinstance(e[2], rdflib.term.URIRef):
o_value = str(e[2])
else:
o_value = str(e[2])
o_datatype = ""
o_language = ""
yield Triple(s=s_value, p=p_value, o=o_value)
if isinstance(e[2], rdflib.term.Literal):
if e[2].language:
o_language = str(e[2].language)
if e[2].datatype:
o_datatype = str(e[2].datatype)
yield Triple(
s=s_value, p=p_value, o=o_value,
o_datatype=o_datatype, o_language=o_language,
)
def run(self):
"""Load triples using Python API"""