mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-07 04:12:10 +02:00
* feat: streaming N-Quads serializer for wire-format triples Groundwork for Phase 2 of #877 (knowledge export). Hand-rolled N-Triples term encoding: rdflib's term.n3() emits Turtle-style forms (numeric shorthand, unescaped newlines) that are invalid in line-oriented N-Quads, so literals are escaped per the ECHAR grammar and IRIs validated for representability. Round-trip tests parse the output back with rdflib's nquads parser and compare term-for-term. * feat: export/import workspace knowledge in .tgx bundles (#877) Phase 2 of the workspace bundle commands: tg-export-workspace now includes the workspace's knowledge by default — per-collection knowledge-graph triples as N-Quads (the collection names the graph, streamed through a tempfile so memory stays flat regardless of knowledge-base size) and the document library (metadata plus content, fetched one document at a time). --config-only skips knowledge on both sides; --triples-limit bounds very large graphs; -f/--flow-id selects the flow the triples services run through. tg-import-workspace streams triples back through the bulk import per collection and recreates library documents (children after parents). Knowledge import is additive, unlike config's skip-existing semantics. Embedding vectors are not carried in bundles: --process re-runs imported documents through the flow, which regenerates extraction output and embeddings; --process-collection targets it. Round-trip covered by unit tests over real archives: export with a mocked Api, re-import, and assert the bulk triples stream and library add calls reproduce the original values (including datatyped literals via the N-Quads path).
This commit is contained in:
parent
68e816e65c
commit
cfbd5b9079
6 changed files with 1040 additions and 181 deletions
|
|
@ -45,4 +45,18 @@ def sample_metadata():
|
|||
"metadata": [],
|
||||
"user": "test-user",
|
||||
"collection": "test-collection"
|
||||
}
|
||||
}
|
||||
|
||||
def iri(v):
|
||||
"""Wire-format IRI term dict, as triples_query_stream yields them."""
|
||||
return {"t": "i", "i": v}
|
||||
|
||||
|
||||
def lit(v, d=None, lang=None):
|
||||
"""Wire-format literal term dict (optional datatype / language)."""
|
||||
t = {"t": "l", "v": v}
|
||||
if d:
|
||||
t["d"] = d
|
||||
if lang:
|
||||
t["l"] = lang
|
||||
return t
|
||||
|
|
|
|||
97
tests/unit/test_cli/test_nquads.py
Normal file
97
tests/unit/test_cli/test_nquads.py
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
"""
|
||||
Round-trip tests for the streaming N-Quads serializer: wire-format triples
|
||||
are serialized line-by-line, then parsed back with rdflib's nquads parser
|
||||
and compared term-for-term — proving the output is valid N-Quads and the
|
||||
encoding (escaping, datatypes, language tags, unicode) is lossless.
|
||||
"""
|
||||
|
||||
import io
|
||||
|
||||
import rdflib
|
||||
|
||||
from trustgraph.cli.nquads import serialize_nquads, triple_to_nquad
|
||||
|
||||
from tests.unit.test_cli.conftest import iri, lit
|
||||
|
||||
GRAPH = "urn:trustgraph:collection:default"
|
||||
|
||||
|
||||
def roundtrip(batches):
|
||||
"""Serialize then parse back; return (parsed_dataset, written, skipped)."""
|
||||
out = io.StringIO()
|
||||
written, skipped = serialize_nquads(batches, GRAPH, out)
|
||||
ds = rdflib.Dataset()
|
||||
ds.parse(data=out.getvalue(), format="nquads")
|
||||
return ds, written, skipped
|
||||
|
||||
|
||||
class TestNquadsRoundTrip:
|
||||
|
||||
def test_iri_and_literal_flavours_survive_roundtrip(self):
|
||||
batches = [[
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
|
||||
"o": iri("http://example.com/o")},
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/label"),
|
||||
"o": lit("plain value")},
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/label"),
|
||||
"o": lit("bonjour", lang="fr")},
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/count"),
|
||||
"o": lit("42", d="http://www.w3.org/2001/XMLSchema#integer")},
|
||||
]]
|
||||
ds, written, skipped = roundtrip(batches)
|
||||
|
||||
assert (written, skipped) == (4, 0)
|
||||
quads = list(ds.quads((None, None, None, None)))
|
||||
assert len(quads) == 4
|
||||
g = rdflib.URIRef(GRAPH)
|
||||
assert all(q[3] == g for q in quads)
|
||||
|
||||
objects = {q[2] for q in quads}
|
||||
assert rdflib.URIRef("http://example.com/o") in objects
|
||||
assert rdflib.Literal("plain value") in objects
|
||||
assert rdflib.Literal("bonjour", lang="fr") in objects
|
||||
assert rdflib.Literal(
|
||||
"42", datatype=rdflib.URIRef("http://www.w3.org/2001/XMLSchema#integer")
|
||||
) in objects
|
||||
|
||||
def test_hostile_literal_content_is_escaped_losslessly(self):
|
||||
nasty = 'line1\nline2\t"quoted" back\\slash 中文 emoji\U0001f680'
|
||||
batches = [[{
|
||||
"s": iri("http://example.com/s"),
|
||||
"p": iri("http://example.com/note"),
|
||||
"o": lit(nasty),
|
||||
}]]
|
||||
ds, written, skipped = roundtrip(batches)
|
||||
|
||||
assert (written, skipped) == (1, 0)
|
||||
obj = next(iter(ds.quads((None, None, None, None))))[2]
|
||||
assert str(obj) == nasty
|
||||
|
||||
def test_malformed_and_unrepresentable_terms_are_skipped_not_emitted(self):
|
||||
batches = [[
|
||||
# IRI with a space (matches graph_to_turtle's malformed skip)
|
||||
{"s": iri("http://example.com/bad iri"), "p": iri("http://example.com/p"),
|
||||
"o": lit("x")},
|
||||
# RDF-star quoted triple: no N-Quads encoding
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
|
||||
"o": {"t": "r", "r": {}}},
|
||||
# literal in predicate position: invalid RDF
|
||||
{"s": iri("http://example.com/s"), "p": lit("not-a-predicate"),
|
||||
"o": lit("x")},
|
||||
# one good triple to prove the stream continues past skips
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
|
||||
"o": lit("good")},
|
||||
]]
|
||||
ds, written, skipped = roundtrip(batches)
|
||||
|
||||
assert (written, skipped) == (1, 3)
|
||||
assert len(list(ds.quads((None, None, None, None)))) == 1
|
||||
|
||||
def test_streaming_shape_one_line_per_triple(self):
|
||||
line = triple_to_nquad(
|
||||
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
|
||||
"o": lit("v")},
|
||||
f"<{GRAPH}>",
|
||||
)
|
||||
assert line.endswith(" .\n")
|
||||
assert line.count("\n") == 1
|
||||
|
|
@ -3,19 +3,25 @@ Tests for tg-export-workspace / tg-import-workspace (.tgx bundle commands).
|
|||
|
||||
The Api class is mocked in each command module's namespace (same pattern as
|
||||
test_config_commands.py); bundles are written to and read from tmp_path so
|
||||
the archive format itself is exercised end-to-end.
|
||||
the archive format itself is exercised end-to-end, including the Phase-2
|
||||
knowledge tree (per-collection N-Quads + library documents).
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import io
|
||||
import json
|
||||
import tarfile
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from trustgraph.api.types import ConfigValue
|
||||
from trustgraph.api.types import ConfigValue, Triple
|
||||
from trustgraph.cli.export_workspace import export_workspace
|
||||
from trustgraph.cli.import_workspace import import_workspace
|
||||
|
||||
from tests.unit.test_cli.conftest import iri, lit
|
||||
|
||||
SAMPLE_CONFIG = {
|
||||
"prompt": {
|
||||
"extract-concepts": json.dumps({"template": "Extract {{q}}"}),
|
||||
|
|
@ -26,25 +32,110 @@ SAMPLE_CONFIG = {
|
|||
},
|
||||
}
|
||||
|
||||
# Wire-format triples for one collection, incl. a datatyped literal.
|
||||
WIRE_BATCHES = [[
|
||||
{"s": iri("http://ex.com/s"), "p": iri("http://ex.com/p"),
|
||||
"o": iri("http://ex.com/o")},
|
||||
{"s": iri("http://ex.com/s"), "p": iri("http://ex.com/count"),
|
||||
"o": lit("42", d="http://www.w3.org/2001/XMLSchema#integer")},
|
||||
]]
|
||||
|
||||
def make_mock_api():
|
||||
mock_config = Mock()
|
||||
DOC = SimpleNamespace(
|
||||
id="doc-1",
|
||||
time=datetime.datetime(2026, 7, 1, 12, 0, 0),
|
||||
kind="text/plain",
|
||||
title="Policy",
|
||||
comments="returns policy",
|
||||
metadata=[Triple(s="http://ex.com/doc-1", p="http://ex.com/about",
|
||||
o="returns")],
|
||||
tags=["policy"],
|
||||
parent_id="",
|
||||
document_type="source",
|
||||
)
|
||||
|
||||
|
||||
def make_mock_api(collections=(), batches=(), docs=(), contents=None):
|
||||
"""Full-surface Api mock; returns (mock_api, mock_config)."""
|
||||
mock_api = Mock()
|
||||
|
||||
mock_config = Mock()
|
||||
mock_api.config.return_value = mock_config
|
||||
mock_config.all.return_value = (SAMPLE_CONFIG, "v42")
|
||||
|
||||
mock_api.collection.return_value.list_collections.return_value = \
|
||||
list(collections)
|
||||
|
||||
flow = mock_api.socket.return_value.flow.return_value
|
||||
flow.triples_query_stream.return_value = iter(batches)
|
||||
|
||||
library = mock_api.library.return_value
|
||||
library.get_documents.return_value = list(docs)
|
||||
library.get_document_content.side_effect = \
|
||||
lambda id: (contents or {}).get(id, b"")
|
||||
|
||||
return mock_api, mock_config
|
||||
|
||||
|
||||
def export_bundle(path, collections=(), batches=(), docs=(), contents=None,
|
||||
**kwargs):
|
||||
"""Export SAMPLE_CONFIG (+ optional knowledge mocks) to path."""
|
||||
mock_api, mock_config = make_mock_api(
|
||||
collections=collections, batches=batches, docs=docs,
|
||||
contents=contents,
|
||||
)
|
||||
with patch("trustgraph.cli.export_workspace.Api") as api_cls:
|
||||
api_cls.return_value = mock_api
|
||||
export_workspace(
|
||||
url="http://api/", workspace="source-ws", output=str(path),
|
||||
**kwargs,
|
||||
)
|
||||
return mock_api, mock_config
|
||||
|
||||
|
||||
DEFAULT_MANIFEST = {
|
||||
"format": "tgx", "format_version": 1, "workspace": "w",
|
||||
"contents": {"config": True, "knowledge": True},
|
||||
}
|
||||
|
||||
|
||||
def write_bundle(path, members, manifest=DEFAULT_MANIFEST):
|
||||
"""Write a raw .tgx from name -> bytes (manifest added first)."""
|
||||
entries = {"manifest.json": json.dumps(manifest).encode(), **members}
|
||||
with tarfile.open(path, "w:gz") as tar:
|
||||
for name, data in entries.items():
|
||||
info = tarfile.TarInfo(name)
|
||||
info.size = len(data)
|
||||
tar.addfile(info, io.BytesIO(data))
|
||||
return path
|
||||
|
||||
|
||||
def run_import(mock_api, input, **kwargs):
|
||||
"""Run import_workspace against a mocked Api."""
|
||||
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
|
||||
api_cls.return_value = mock_api
|
||||
import_workspace(url="http://api/", input=str(input), **kwargs)
|
||||
return api_cls
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def bundle(tmp_path):
|
||||
"""Export SAMPLE_CONFIG to a real .tgx and yield its path."""
|
||||
"""Config-only-shaped bundle (no collections/docs mocked)."""
|
||||
path = tmp_path / "ws.tgx"
|
||||
with patch("trustgraph.cli.export_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
mock_config.all.return_value = (SAMPLE_CONFIG, "v42")
|
||||
export_workspace(
|
||||
url="http://api/", workspace="source-ws", output=str(path),
|
||||
)
|
||||
export_bundle(path)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def knowledge_bundle(tmp_path):
|
||||
"""Bundle with one collection (2 triples) and one document."""
|
||||
path = tmp_path / "kws.tgx"
|
||||
export_bundle(
|
||||
path,
|
||||
collections=[SimpleNamespace(collection="research")],
|
||||
batches=WIRE_BATCHES,
|
||||
docs=[DOC],
|
||||
contents={"doc-1": b"Customers may return items."},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
|
|
@ -58,7 +149,10 @@ class TestExportWorkspace:
|
|||
assert manifest["format"] == "tgx"
|
||||
assert manifest["workspace"] == "source-ws"
|
||||
assert manifest["config_version"] == "v42"
|
||||
assert manifest["contents"] == {"config": True, "knowledge": False}
|
||||
assert manifest["contents"] == {"config": True, "knowledge": True}
|
||||
assert manifest["knowledge"] == {
|
||||
"collections": [], "documents": 0, "triples": {},
|
||||
}
|
||||
|
||||
assert "config/prompt/extract-concepts.json" in names
|
||||
assert "config/prompt/answer.json" in names
|
||||
|
|
@ -79,12 +173,12 @@ class TestExportWorkspace:
|
|||
|
||||
def test_path_unsafe_keys_are_quoted_in_filenames(self, tmp_path):
|
||||
path = tmp_path / "ws.tgx"
|
||||
mock_api, mock_config = make_mock_api()
|
||||
mock_config.all.return_value = (
|
||||
{"prompt": {"a/b": json.dumps({"x": 1})}}, "v1",
|
||||
)
|
||||
with patch("trustgraph.cli.export_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
mock_config.all.return_value = (
|
||||
{"prompt": {"a/b": json.dumps({"x": 1})}}, "v1",
|
||||
)
|
||||
export_workspace(
|
||||
url="http://api/", workspace="ws", output=str(path),
|
||||
)
|
||||
|
|
@ -94,16 +188,63 @@ class TestExportWorkspace:
|
|||
assert "config/prompt/a%2Fb.json" in names
|
||||
assert entry["key"] == "a/b"
|
||||
|
||||
def test_knowledge_tree_written_per_collection_and_document(
|
||||
self, knowledge_bundle):
|
||||
with tarfile.open(knowledge_bundle, "r:gz") as tar:
|
||||
names = tar.getnames()
|
||||
manifest = json.load(tar.extractfile("manifest.json"))
|
||||
nq = tar.extractfile(
|
||||
"knowledge/research/triples.nq").read().decode()
|
||||
meta = json.load(
|
||||
tar.extractfile("knowledge/library/doc-1.meta.json"))
|
||||
content = tar.extractfile(
|
||||
"knowledge/library/doc-1.content").read()
|
||||
|
||||
assert manifest["contents"]["knowledge"] is True
|
||||
assert manifest["knowledge"] == {
|
||||
"collections": ["research"], "documents": 1,
|
||||
"triples": {"research": 2},
|
||||
}
|
||||
assert "knowledge/research/triples.nq" in names
|
||||
# N-Quads: one line per triple, graph = the collection IRI, and
|
||||
# the datatyped literal keeps its full quoted form.
|
||||
lines = [ln for ln in nq.splitlines() if ln]
|
||||
assert len(lines) == 2
|
||||
assert all("<urn:trustgraph:collection:research>" in ln
|
||||
for ln in lines)
|
||||
assert '"42"^^<http://www.w3.org/2001/XMLSchema#integer>' in nq
|
||||
|
||||
assert meta["id"] == "doc-1"
|
||||
assert meta["title"] == "Policy"
|
||||
assert meta["metadata"] == [{
|
||||
"s": "http://ex.com/doc-1", "p": "http://ex.com/about",
|
||||
"o": "returns",
|
||||
}]
|
||||
assert content == b"Customers may return items."
|
||||
|
||||
def test_config_only_skips_knowledge(self, tmp_path):
|
||||
path = tmp_path / "co.tgx"
|
||||
mock_api, _ = export_bundle(
|
||||
path,
|
||||
collections=[SimpleNamespace(collection="research")],
|
||||
config_only=True,
|
||||
)
|
||||
with tarfile.open(path, "r:gz") as tar:
|
||||
names = tar.getnames()
|
||||
manifest = json.load(tar.extractfile("manifest.json"))
|
||||
assert manifest["contents"] == {"config": True, "knowledge": False}
|
||||
assert "knowledge" not in manifest
|
||||
assert not any(n.startswith("knowledge/") for n in names)
|
||||
mock_api.collection.return_value.list_collections.assert_not_called()
|
||||
mock_api.socket.assert_not_called()
|
||||
mock_api.library.assert_not_called()
|
||||
|
||||
|
||||
class TestImportWorkspace:
|
||||
|
||||
def test_roundtrip_puts_all_values_with_overwrite(self, bundle):
|
||||
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
import_workspace(
|
||||
url="http://api/", input=str(bundle), overwrite=True,
|
||||
)
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls = run_import(mock_api, bundle, overwrite=True)
|
||||
|
||||
# Target workspace defaults to the manifest's workspace.
|
||||
api_cls.assert_called_once_with(
|
||||
|
|
@ -123,27 +264,22 @@ class TestImportWorkspace:
|
|||
assert all(isinstance(v, ConfigValue) for v in values)
|
||||
|
||||
def test_workspace_flag_renames_target(self, bundle):
|
||||
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
import_workspace(
|
||||
url="http://api/", input=str(bundle), workspace="staging",
|
||||
overwrite=True,
|
||||
)
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls = run_import(
|
||||
mock_api, bundle, workspace="staging", overwrite=True,
|
||||
)
|
||||
api_cls.assert_called_once_with(
|
||||
"http://api/", token=None, workspace="staging",
|
||||
)
|
||||
|
||||
def test_default_skips_existing_keys(self, bundle):
|
||||
"""WorkspaceInit re-run semantics: only missing keys are written."""
|
||||
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
mock_config.list.side_effect = lambda t: {
|
||||
"prompt": ["extract-concepts"],
|
||||
"tool": [],
|
||||
}[t]
|
||||
import_workspace(url="http://api/", input=str(bundle))
|
||||
mock_api, mock_config = make_mock_api()
|
||||
mock_config.list.side_effect = lambda t: {
|
||||
"prompt": ["extract-concepts"],
|
||||
"tool": [],
|
||||
}[t]
|
||||
run_import(mock_api, bundle)
|
||||
|
||||
values = mock_config.put.call_args.args[0]
|
||||
assert sorted((v.type, v.key) for v in values) == [
|
||||
|
|
@ -152,17 +288,11 @@ class TestImportWorkspace:
|
|||
]
|
||||
|
||||
def test_dry_run_writes_nothing(self, bundle, capsys):
|
||||
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
import_workspace(
|
||||
url="http://api/", input=str(bundle), overwrite=True,
|
||||
dry_run=True,
|
||||
)
|
||||
mock_api, mock_config = make_mock_api()
|
||||
run_import(mock_api, bundle, overwrite=True, dry_run=True)
|
||||
mock_config.put.assert_not_called()
|
||||
out = capsys.readouterr().out
|
||||
assert "would import prompt/extract-concepts" in out
|
||||
assert "3 item(s) would be imported" in out
|
||||
|
||||
def test_rejects_bundle_without_manifest(self, tmp_path):
|
||||
path = tmp_path / "bad.tgx"
|
||||
|
|
@ -173,52 +303,147 @@ class TestImportWorkspace:
|
|||
import_workspace(url="http://api/", input=str(path))
|
||||
|
||||
def test_rejects_newer_format_version(self, tmp_path):
|
||||
import io
|
||||
path = tmp_path / "future.tgx"
|
||||
manifest = json.dumps({
|
||||
"format": "tgx", "format_version": 99, "workspace": "w",
|
||||
"contents": {"config": True, "knowledge": False},
|
||||
}).encode()
|
||||
with tarfile.open(path, "w:gz") as tar:
|
||||
info = tarfile.TarInfo("manifest.json")
|
||||
info.size = len(manifest)
|
||||
tar.addfile(info, io.BytesIO(manifest))
|
||||
path = write_bundle(
|
||||
tmp_path / "future.tgx", {},
|
||||
manifest={**DEFAULT_MANIFEST, "format_version": 99},
|
||||
)
|
||||
with patch("trustgraph.cli.import_workspace.Api"):
|
||||
with pytest.raises(RuntimeError, match="newer than this tool"):
|
||||
import_workspace(url="http://api/", input=str(path))
|
||||
|
||||
def test_refuses_knowledge_bundle_without_config_only(self, tmp_path):
|
||||
import io
|
||||
path = tmp_path / "knowledge.tgx"
|
||||
manifest = json.dumps({
|
||||
"format": "tgx", "format_version": 1, "workspace": "w",
|
||||
"contents": {"config": True, "knowledge": True},
|
||||
}).encode()
|
||||
with tarfile.open(path, "w:gz") as tar:
|
||||
info = tarfile.TarInfo("manifest.json")
|
||||
info.size = len(manifest)
|
||||
tar.addfile(info, io.BytesIO(manifest))
|
||||
with patch("trustgraph.cli.import_workspace.Api"):
|
||||
with pytest.raises(RuntimeError, match="--config-only"):
|
||||
import_workspace(url="http://api/", input=str(path))
|
||||
|
||||
def test_config_only_flag_allows_knowledge_bundle(self, tmp_path):
|
||||
import io
|
||||
path = tmp_path / "knowledge.tgx"
|
||||
manifest = json.dumps({
|
||||
"format": "tgx", "format_version": 1, "workspace": "w",
|
||||
"contents": {"config": True, "knowledge": True},
|
||||
}).encode()
|
||||
with tarfile.open(path, "w:gz") as tar:
|
||||
info = tarfile.TarInfo("manifest.json")
|
||||
info.size = len(manifest)
|
||||
tar.addfile(info, io.BytesIO(manifest))
|
||||
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
|
||||
mock_api, mock_config = make_mock_api()
|
||||
api_cls.return_value = mock_api
|
||||
import_workspace(
|
||||
url="http://api/", input=str(path), config_only=True,
|
||||
overwrite=True,
|
||||
)
|
||||
# No config entries in this bundle; nothing written, no error.
|
||||
mock_config.put.assert_not_called()
|
||||
class TestImportKnowledge:
|
||||
|
||||
def test_roundtrip_imports_triples_and_documents(self, knowledge_bundle):
|
||||
mock_api, mock_config = make_mock_api()
|
||||
run_import(mock_api, knowledge_bundle, overwrite=True)
|
||||
|
||||
# Triples land in the bulk import stream for the right collection.
|
||||
bulk = mock_api.bulk.return_value
|
||||
call = bulk.import_triples.call_args
|
||||
assert call.args[0] == "default" # flow id
|
||||
triples = sorted(list(call.args[1]), key=lambda t: t.p)
|
||||
assert triples == [
|
||||
Triple(s="http://ex.com/s", p="http://ex.com/count", o="42"),
|
||||
Triple(s="http://ex.com/s", p="http://ex.com/p",
|
||||
o="http://ex.com/o"),
|
||||
]
|
||||
assert call.kwargs["metadata"]["collection"] == "research"
|
||||
|
||||
# The document is recreated with its metadata and content.
|
||||
add = mock_api.library.return_value.add_document.call_args
|
||||
assert add.kwargs["id"] == "doc-1"
|
||||
assert add.kwargs["document"] == b"Customers may return items."
|
||||
assert add.kwargs["title"] == "Policy"
|
||||
assert add.kwargs["kind"] == "text/plain"
|
||||
assert add.kwargs["tags"] == ["policy"]
|
||||
assert add.kwargs["metadata"] == [
|
||||
Triple(s="http://ex.com/doc-1", p="http://ex.com/about",
|
||||
o="returns"),
|
||||
]
|
||||
# No processing unless asked: embeddings re-derivation is opt-in.
|
||||
mock_api.library.return_value.start_processing.assert_not_called()
|
||||
|
||||
def test_config_only_skips_knowledge_on_import(self, knowledge_bundle):
|
||||
mock_api, mock_config = make_mock_api()
|
||||
run_import(mock_api, knowledge_bundle, overwrite=True,
|
||||
config_only=True)
|
||||
mock_api.bulk.assert_not_called()
|
||||
mock_api.library.return_value.add_document.assert_not_called()
|
||||
mock_config.put.assert_called_once()
|
||||
|
||||
def test_dry_run_covers_knowledge(self, knowledge_bundle, capsys):
|
||||
mock_api, mock_config = make_mock_api()
|
||||
run_import(mock_api, knowledge_bundle, overwrite=True, dry_run=True)
|
||||
mock_api.bulk.assert_not_called()
|
||||
mock_api.library.return_value.add_document.assert_not_called()
|
||||
out = capsys.readouterr().out
|
||||
assert "would import 2 triple(s) into collection 'research'" in out
|
||||
assert "would import document doc-1" in out
|
||||
|
||||
def test_process_flag_reprocesses_documents(self, knowledge_bundle):
|
||||
mock_api, mock_config = make_mock_api()
|
||||
run_import(mock_api, knowledge_bundle, overwrite=True, process=True,
|
||||
process_collection="research")
|
||||
proc = mock_api.library.return_value.start_processing.call_args
|
||||
assert proc.kwargs["document_id"] == "doc-1"
|
||||
assert proc.kwargs["flow"] == "default"
|
||||
assert proc.kwargs["collection"] == "research"
|
||||
|
||||
|
||||
class TestImportDocumentSkipOverwrite:
|
||||
"""Live-verified semantics: existing documents skip by default, replace
|
||||
with --overwrite (remove + add; the library API has no content update)."""
|
||||
|
||||
def _bundle_with_doc(self, tmp_path):
|
||||
return write_bundle(tmp_path / "doc.tgx", {
|
||||
"knowledge/library/doc-1.meta.json": json.dumps(
|
||||
{"id": "doc-1", "title": "T", "metadata": []}).encode(),
|
||||
"knowledge/library/doc-1.content": b"hello",
|
||||
})
|
||||
|
||||
def test_existing_document_is_skipped_not_fatal(self, tmp_path):
|
||||
path = self._bundle_with_doc(tmp_path)
|
||||
mock_api, mock_config = make_mock_api(
|
||||
docs=[SimpleNamespace(id="doc-1")],
|
||||
)
|
||||
run_import(mock_api, path, overwrite=False)
|
||||
lib = mock_api.library.return_value
|
||||
lib.add_document.assert_not_called()
|
||||
lib.remove_document.assert_not_called()
|
||||
|
||||
def test_overwrite_replaces_existing_document(self, tmp_path):
|
||||
path = self._bundle_with_doc(tmp_path)
|
||||
mock_api, mock_config = make_mock_api(
|
||||
docs=[SimpleNamespace(id="doc-1")],
|
||||
)
|
||||
run_import(mock_api, path, overwrite=True)
|
||||
lib = mock_api.library.return_value
|
||||
lib.remove_document.assert_called_once_with("doc-1")
|
||||
lib.add_document.assert_called_once()
|
||||
|
||||
|
||||
class TestCollectionDiscovery:
|
||||
"""Live-verified: implicitly-created collections (raw triple loads) are
|
||||
queryable but unlisted, so export merges --collection extras and import
|
||||
registers what it restores."""
|
||||
|
||||
def test_export_includes_extra_unregistered_collections(self, tmp_path):
|
||||
path = tmp_path / "ws.tgx"
|
||||
mock_api, _ = export_bundle(
|
||||
path,
|
||||
collections=[SimpleNamespace(collection="default")],
|
||||
batches=[[]],
|
||||
extra_collections=["research"],
|
||||
)
|
||||
with tarfile.open(path, "r:gz") as tar:
|
||||
manifest = json.load(tar.extractfile("manifest.json"))
|
||||
assert manifest["knowledge"]["collections"] == ["default", "research"]
|
||||
|
||||
def test_import_registers_each_restored_collection(self, tmp_path):
|
||||
path = write_bundle(tmp_path / "kb.tgx", {
|
||||
"knowledge/research/triples.nq": (
|
||||
b'<http://ex.com/s> <http://ex.com/p> "v" '
|
||||
b'<urn:trustgraph:collection:research> .\n'
|
||||
),
|
||||
})
|
||||
mock_api, mock_config = make_mock_api()
|
||||
run_import(mock_api, path)
|
||||
mock_api.collection.return_value.update_collection.assert_called_once_with(
|
||||
"research", name="research",
|
||||
)
|
||||
|
||||
def test_registered_collections_are_not_reregistered(self, tmp_path):
|
||||
"""update_collection is an upsert that clears omitted fields, so a
|
||||
collection already in the registry must be left untouched."""
|
||||
path = write_bundle(tmp_path / "kb.tgx", {
|
||||
"knowledge/research/triples.nq": (
|
||||
b'<http://ex.com/s> <http://ex.com/p> "v" '
|
||||
b'<urn:trustgraph:collection:research> .\n'
|
||||
),
|
||||
})
|
||||
mock_api, mock_config = make_mock_api(
|
||||
collections=[SimpleNamespace(collection="research")],
|
||||
)
|
||||
run_import(mock_api, path)
|
||||
mock_api.collection.return_value.update_collection.assert_not_called()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue