feat: export/import workspace knowledge in .tgx bundles (#877 Phase 2) (#1024)

* feat: streaming N-Quads serializer for wire-format triples

Groundwork for Phase 2 of #877 (knowledge export). Hand-rolled
N-Triples term encoding: rdflib's term.n3() emits Turtle-style forms
(numeric shorthand, unescaped newlines) that are invalid in
line-oriented N-Quads, so literals are escaped per the ECHAR grammar
and IRIs validated for representability. Round-trip tests parse the
output back with rdflib's nquads parser and compare term-for-term.

* feat: export/import workspace knowledge in .tgx bundles (#877)

Phase 2 of the workspace bundle commands: tg-export-workspace now
includes the workspace's knowledge by default — per-collection
knowledge-graph triples as N-Quads (the collection names the graph,
streamed through a tempfile so memory stays flat regardless of
knowledge-base size) and the document library (metadata plus content,
fetched one document at a time). --config-only skips knowledge on both
sides; --triples-limit bounds very large graphs; -f/--flow-id selects
the flow the triples services run through.

tg-import-workspace streams triples back through the bulk import per
collection and recreates library documents (children after parents).
Knowledge import is additive, unlike config's skip-existing semantics.
Embedding vectors are not carried in bundles: --process re-runs
imported documents through the flow, which regenerates extraction
output and embeddings; --process-collection targets it.

Round-trip covered by unit tests over real archives: export with a
mocked Api, re-import, and assert the bulk triples stream and library
add calls reproduce the original values (including datatyped literals
via the N-Quads path).
This commit is contained in:
Sunny Yang 2026-07-06 03:45:23 -06:00 committed by GitHub
parent 68e816e65c
commit cfbd5b9079
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 1040 additions and 181 deletions

View file

@ -45,4 +45,18 @@ def sample_metadata():
"metadata": [],
"user": "test-user",
"collection": "test-collection"
}
}
def iri(v):
"""Wire-format IRI term dict, as triples_query_stream yields them."""
return {"t": "i", "i": v}
def lit(v, d=None, lang=None):
"""Wire-format literal term dict (optional datatype / language)."""
t = {"t": "l", "v": v}
if d:
t["d"] = d
if lang:
t["l"] = lang
return t

View file

@ -0,0 +1,97 @@
"""
Round-trip tests for the streaming N-Quads serializer: wire-format triples
are serialized line-by-line, then parsed back with rdflib's nquads parser
and compared term-for-term proving the output is valid N-Quads and the
encoding (escaping, datatypes, language tags, unicode) is lossless.
"""
import io
import rdflib
from trustgraph.cli.nquads import serialize_nquads, triple_to_nquad
from tests.unit.test_cli.conftest import iri, lit
GRAPH = "urn:trustgraph:collection:default"
def roundtrip(batches):
"""Serialize then parse back; return (parsed_dataset, written, skipped)."""
out = io.StringIO()
written, skipped = serialize_nquads(batches, GRAPH, out)
ds = rdflib.Dataset()
ds.parse(data=out.getvalue(), format="nquads")
return ds, written, skipped
class TestNquadsRoundTrip:
def test_iri_and_literal_flavours_survive_roundtrip(self):
batches = [[
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
"o": iri("http://example.com/o")},
{"s": iri("http://example.com/s"), "p": iri("http://example.com/label"),
"o": lit("plain value")},
{"s": iri("http://example.com/s"), "p": iri("http://example.com/label"),
"o": lit("bonjour", lang="fr")},
{"s": iri("http://example.com/s"), "p": iri("http://example.com/count"),
"o": lit("42", d="http://www.w3.org/2001/XMLSchema#integer")},
]]
ds, written, skipped = roundtrip(batches)
assert (written, skipped) == (4, 0)
quads = list(ds.quads((None, None, None, None)))
assert len(quads) == 4
g = rdflib.URIRef(GRAPH)
assert all(q[3] == g for q in quads)
objects = {q[2] for q in quads}
assert rdflib.URIRef("http://example.com/o") in objects
assert rdflib.Literal("plain value") in objects
assert rdflib.Literal("bonjour", lang="fr") in objects
assert rdflib.Literal(
"42", datatype=rdflib.URIRef("http://www.w3.org/2001/XMLSchema#integer")
) in objects
def test_hostile_literal_content_is_escaped_losslessly(self):
nasty = 'line1\nline2\t"quoted" back\\slash 中文 emoji\U0001f680'
batches = [[{
"s": iri("http://example.com/s"),
"p": iri("http://example.com/note"),
"o": lit(nasty),
}]]
ds, written, skipped = roundtrip(batches)
assert (written, skipped) == (1, 0)
obj = next(iter(ds.quads((None, None, None, None))))[2]
assert str(obj) == nasty
def test_malformed_and_unrepresentable_terms_are_skipped_not_emitted(self):
batches = [[
# IRI with a space (matches graph_to_turtle's malformed skip)
{"s": iri("http://example.com/bad iri"), "p": iri("http://example.com/p"),
"o": lit("x")},
# RDF-star quoted triple: no N-Quads encoding
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
"o": {"t": "r", "r": {}}},
# literal in predicate position: invalid RDF
{"s": iri("http://example.com/s"), "p": lit("not-a-predicate"),
"o": lit("x")},
# one good triple to prove the stream continues past skips
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
"o": lit("good")},
]]
ds, written, skipped = roundtrip(batches)
assert (written, skipped) == (1, 3)
assert len(list(ds.quads((None, None, None, None)))) == 1
def test_streaming_shape_one_line_per_triple(self):
line = triple_to_nquad(
{"s": iri("http://example.com/s"), "p": iri("http://example.com/p"),
"o": lit("v")},
f"<{GRAPH}>",
)
assert line.endswith(" .\n")
assert line.count("\n") == 1

View file

@ -3,19 +3,25 @@ Tests for tg-export-workspace / tg-import-workspace (.tgx bundle commands).
The Api class is mocked in each command module's namespace (same pattern as
test_config_commands.py); bundles are written to and read from tmp_path so
the archive format itself is exercised end-to-end.
the archive format itself is exercised end-to-end, including the Phase-2
knowledge tree (per-collection N-Quads + library documents).
"""
import datetime
import io
import json
import tarfile
from types import SimpleNamespace
from unittest.mock import Mock, patch
import pytest
from trustgraph.api.types import ConfigValue
from trustgraph.api.types import ConfigValue, Triple
from trustgraph.cli.export_workspace import export_workspace
from trustgraph.cli.import_workspace import import_workspace
from tests.unit.test_cli.conftest import iri, lit
SAMPLE_CONFIG = {
"prompt": {
"extract-concepts": json.dumps({"template": "Extract {{q}}"}),
@ -26,25 +32,110 @@ SAMPLE_CONFIG = {
},
}
# Wire-format triples for one collection, incl. a datatyped literal.
WIRE_BATCHES = [[
{"s": iri("http://ex.com/s"), "p": iri("http://ex.com/p"),
"o": iri("http://ex.com/o")},
{"s": iri("http://ex.com/s"), "p": iri("http://ex.com/count"),
"o": lit("42", d="http://www.w3.org/2001/XMLSchema#integer")},
]]
def make_mock_api():
mock_config = Mock()
DOC = SimpleNamespace(
id="doc-1",
time=datetime.datetime(2026, 7, 1, 12, 0, 0),
kind="text/plain",
title="Policy",
comments="returns policy",
metadata=[Triple(s="http://ex.com/doc-1", p="http://ex.com/about",
o="returns")],
tags=["policy"],
parent_id="",
document_type="source",
)
def make_mock_api(collections=(), batches=(), docs=(), contents=None):
"""Full-surface Api mock; returns (mock_api, mock_config)."""
mock_api = Mock()
mock_config = Mock()
mock_api.config.return_value = mock_config
mock_config.all.return_value = (SAMPLE_CONFIG, "v42")
mock_api.collection.return_value.list_collections.return_value = \
list(collections)
flow = mock_api.socket.return_value.flow.return_value
flow.triples_query_stream.return_value = iter(batches)
library = mock_api.library.return_value
library.get_documents.return_value = list(docs)
library.get_document_content.side_effect = \
lambda id: (contents or {}).get(id, b"")
return mock_api, mock_config
def export_bundle(path, collections=(), batches=(), docs=(), contents=None,
**kwargs):
"""Export SAMPLE_CONFIG (+ optional knowledge mocks) to path."""
mock_api, mock_config = make_mock_api(
collections=collections, batches=batches, docs=docs,
contents=contents,
)
with patch("trustgraph.cli.export_workspace.Api") as api_cls:
api_cls.return_value = mock_api
export_workspace(
url="http://api/", workspace="source-ws", output=str(path),
**kwargs,
)
return mock_api, mock_config
DEFAULT_MANIFEST = {
"format": "tgx", "format_version": 1, "workspace": "w",
"contents": {"config": True, "knowledge": True},
}
def write_bundle(path, members, manifest=DEFAULT_MANIFEST):
"""Write a raw .tgx from name -> bytes (manifest added first)."""
entries = {"manifest.json": json.dumps(manifest).encode(), **members}
with tarfile.open(path, "w:gz") as tar:
for name, data in entries.items():
info = tarfile.TarInfo(name)
info.size = len(data)
tar.addfile(info, io.BytesIO(data))
return path
def run_import(mock_api, input, **kwargs):
"""Run import_workspace against a mocked Api."""
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
api_cls.return_value = mock_api
import_workspace(url="http://api/", input=str(input), **kwargs)
return api_cls
@pytest.fixture
def bundle(tmp_path):
"""Export SAMPLE_CONFIG to a real .tgx and yield its path."""
"""Config-only-shaped bundle (no collections/docs mocked)."""
path = tmp_path / "ws.tgx"
with patch("trustgraph.cli.export_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
mock_config.all.return_value = (SAMPLE_CONFIG, "v42")
export_workspace(
url="http://api/", workspace="source-ws", output=str(path),
)
export_bundle(path)
return path
@pytest.fixture
def knowledge_bundle(tmp_path):
"""Bundle with one collection (2 triples) and one document."""
path = tmp_path / "kws.tgx"
export_bundle(
path,
collections=[SimpleNamespace(collection="research")],
batches=WIRE_BATCHES,
docs=[DOC],
contents={"doc-1": b"Customers may return items."},
)
return path
@ -58,7 +149,10 @@ class TestExportWorkspace:
assert manifest["format"] == "tgx"
assert manifest["workspace"] == "source-ws"
assert manifest["config_version"] == "v42"
assert manifest["contents"] == {"config": True, "knowledge": False}
assert manifest["contents"] == {"config": True, "knowledge": True}
assert manifest["knowledge"] == {
"collections": [], "documents": 0, "triples": {},
}
assert "config/prompt/extract-concepts.json" in names
assert "config/prompt/answer.json" in names
@ -79,12 +173,12 @@ class TestExportWorkspace:
def test_path_unsafe_keys_are_quoted_in_filenames(self, tmp_path):
path = tmp_path / "ws.tgx"
mock_api, mock_config = make_mock_api()
mock_config.all.return_value = (
{"prompt": {"a/b": json.dumps({"x": 1})}}, "v1",
)
with patch("trustgraph.cli.export_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
mock_config.all.return_value = (
{"prompt": {"a/b": json.dumps({"x": 1})}}, "v1",
)
export_workspace(
url="http://api/", workspace="ws", output=str(path),
)
@ -94,16 +188,63 @@ class TestExportWorkspace:
assert "config/prompt/a%2Fb.json" in names
assert entry["key"] == "a/b"
def test_knowledge_tree_written_per_collection_and_document(
self, knowledge_bundle):
with tarfile.open(knowledge_bundle, "r:gz") as tar:
names = tar.getnames()
manifest = json.load(tar.extractfile("manifest.json"))
nq = tar.extractfile(
"knowledge/research/triples.nq").read().decode()
meta = json.load(
tar.extractfile("knowledge/library/doc-1.meta.json"))
content = tar.extractfile(
"knowledge/library/doc-1.content").read()
assert manifest["contents"]["knowledge"] is True
assert manifest["knowledge"] == {
"collections": ["research"], "documents": 1,
"triples": {"research": 2},
}
assert "knowledge/research/triples.nq" in names
# N-Quads: one line per triple, graph = the collection IRI, and
# the datatyped literal keeps its full quoted form.
lines = [ln for ln in nq.splitlines() if ln]
assert len(lines) == 2
assert all("<urn:trustgraph:collection:research>" in ln
for ln in lines)
assert '"42"^^<http://www.w3.org/2001/XMLSchema#integer>' in nq
assert meta["id"] == "doc-1"
assert meta["title"] == "Policy"
assert meta["metadata"] == [{
"s": "http://ex.com/doc-1", "p": "http://ex.com/about",
"o": "returns",
}]
assert content == b"Customers may return items."
def test_config_only_skips_knowledge(self, tmp_path):
path = tmp_path / "co.tgx"
mock_api, _ = export_bundle(
path,
collections=[SimpleNamespace(collection="research")],
config_only=True,
)
with tarfile.open(path, "r:gz") as tar:
names = tar.getnames()
manifest = json.load(tar.extractfile("manifest.json"))
assert manifest["contents"] == {"config": True, "knowledge": False}
assert "knowledge" not in manifest
assert not any(n.startswith("knowledge/") for n in names)
mock_api.collection.return_value.list_collections.assert_not_called()
mock_api.socket.assert_not_called()
mock_api.library.assert_not_called()
class TestImportWorkspace:
def test_roundtrip_puts_all_values_with_overwrite(self, bundle):
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
import_workspace(
url="http://api/", input=str(bundle), overwrite=True,
)
mock_api, mock_config = make_mock_api()
api_cls = run_import(mock_api, bundle, overwrite=True)
# Target workspace defaults to the manifest's workspace.
api_cls.assert_called_once_with(
@ -123,27 +264,22 @@ class TestImportWorkspace:
assert all(isinstance(v, ConfigValue) for v in values)
def test_workspace_flag_renames_target(self, bundle):
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
import_workspace(
url="http://api/", input=str(bundle), workspace="staging",
overwrite=True,
)
mock_api, mock_config = make_mock_api()
api_cls = run_import(
mock_api, bundle, workspace="staging", overwrite=True,
)
api_cls.assert_called_once_with(
"http://api/", token=None, workspace="staging",
)
def test_default_skips_existing_keys(self, bundle):
"""WorkspaceInit re-run semantics: only missing keys are written."""
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
mock_config.list.side_effect = lambda t: {
"prompt": ["extract-concepts"],
"tool": [],
}[t]
import_workspace(url="http://api/", input=str(bundle))
mock_api, mock_config = make_mock_api()
mock_config.list.side_effect = lambda t: {
"prompt": ["extract-concepts"],
"tool": [],
}[t]
run_import(mock_api, bundle)
values = mock_config.put.call_args.args[0]
assert sorted((v.type, v.key) for v in values) == [
@ -152,17 +288,11 @@ class TestImportWorkspace:
]
def test_dry_run_writes_nothing(self, bundle, capsys):
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
import_workspace(
url="http://api/", input=str(bundle), overwrite=True,
dry_run=True,
)
mock_api, mock_config = make_mock_api()
run_import(mock_api, bundle, overwrite=True, dry_run=True)
mock_config.put.assert_not_called()
out = capsys.readouterr().out
assert "would import prompt/extract-concepts" in out
assert "3 item(s) would be imported" in out
def test_rejects_bundle_without_manifest(self, tmp_path):
path = tmp_path / "bad.tgx"
@ -173,52 +303,147 @@ class TestImportWorkspace:
import_workspace(url="http://api/", input=str(path))
def test_rejects_newer_format_version(self, tmp_path):
import io
path = tmp_path / "future.tgx"
manifest = json.dumps({
"format": "tgx", "format_version": 99, "workspace": "w",
"contents": {"config": True, "knowledge": False},
}).encode()
with tarfile.open(path, "w:gz") as tar:
info = tarfile.TarInfo("manifest.json")
info.size = len(manifest)
tar.addfile(info, io.BytesIO(manifest))
path = write_bundle(
tmp_path / "future.tgx", {},
manifest={**DEFAULT_MANIFEST, "format_version": 99},
)
with patch("trustgraph.cli.import_workspace.Api"):
with pytest.raises(RuntimeError, match="newer than this tool"):
import_workspace(url="http://api/", input=str(path))
def test_refuses_knowledge_bundle_without_config_only(self, tmp_path):
import io
path = tmp_path / "knowledge.tgx"
manifest = json.dumps({
"format": "tgx", "format_version": 1, "workspace": "w",
"contents": {"config": True, "knowledge": True},
}).encode()
with tarfile.open(path, "w:gz") as tar:
info = tarfile.TarInfo("manifest.json")
info.size = len(manifest)
tar.addfile(info, io.BytesIO(manifest))
with patch("trustgraph.cli.import_workspace.Api"):
with pytest.raises(RuntimeError, match="--config-only"):
import_workspace(url="http://api/", input=str(path))
def test_config_only_flag_allows_knowledge_bundle(self, tmp_path):
import io
path = tmp_path / "knowledge.tgx"
manifest = json.dumps({
"format": "tgx", "format_version": 1, "workspace": "w",
"contents": {"config": True, "knowledge": True},
}).encode()
with tarfile.open(path, "w:gz") as tar:
info = tarfile.TarInfo("manifest.json")
info.size = len(manifest)
tar.addfile(info, io.BytesIO(manifest))
with patch("trustgraph.cli.import_workspace.Api") as api_cls:
mock_api, mock_config = make_mock_api()
api_cls.return_value = mock_api
import_workspace(
url="http://api/", input=str(path), config_only=True,
overwrite=True,
)
# No config entries in this bundle; nothing written, no error.
mock_config.put.assert_not_called()
class TestImportKnowledge:
def test_roundtrip_imports_triples_and_documents(self, knowledge_bundle):
mock_api, mock_config = make_mock_api()
run_import(mock_api, knowledge_bundle, overwrite=True)
# Triples land in the bulk import stream for the right collection.
bulk = mock_api.bulk.return_value
call = bulk.import_triples.call_args
assert call.args[0] == "default" # flow id
triples = sorted(list(call.args[1]), key=lambda t: t.p)
assert triples == [
Triple(s="http://ex.com/s", p="http://ex.com/count", o="42"),
Triple(s="http://ex.com/s", p="http://ex.com/p",
o="http://ex.com/o"),
]
assert call.kwargs["metadata"]["collection"] == "research"
# The document is recreated with its metadata and content.
add = mock_api.library.return_value.add_document.call_args
assert add.kwargs["id"] == "doc-1"
assert add.kwargs["document"] == b"Customers may return items."
assert add.kwargs["title"] == "Policy"
assert add.kwargs["kind"] == "text/plain"
assert add.kwargs["tags"] == ["policy"]
assert add.kwargs["metadata"] == [
Triple(s="http://ex.com/doc-1", p="http://ex.com/about",
o="returns"),
]
# No processing unless asked: embeddings re-derivation is opt-in.
mock_api.library.return_value.start_processing.assert_not_called()
def test_config_only_skips_knowledge_on_import(self, knowledge_bundle):
mock_api, mock_config = make_mock_api()
run_import(mock_api, knowledge_bundle, overwrite=True,
config_only=True)
mock_api.bulk.assert_not_called()
mock_api.library.return_value.add_document.assert_not_called()
mock_config.put.assert_called_once()
def test_dry_run_covers_knowledge(self, knowledge_bundle, capsys):
mock_api, mock_config = make_mock_api()
run_import(mock_api, knowledge_bundle, overwrite=True, dry_run=True)
mock_api.bulk.assert_not_called()
mock_api.library.return_value.add_document.assert_not_called()
out = capsys.readouterr().out
assert "would import 2 triple(s) into collection 'research'" in out
assert "would import document doc-1" in out
def test_process_flag_reprocesses_documents(self, knowledge_bundle):
mock_api, mock_config = make_mock_api()
run_import(mock_api, knowledge_bundle, overwrite=True, process=True,
process_collection="research")
proc = mock_api.library.return_value.start_processing.call_args
assert proc.kwargs["document_id"] == "doc-1"
assert proc.kwargs["flow"] == "default"
assert proc.kwargs["collection"] == "research"
class TestImportDocumentSkipOverwrite:
"""Live-verified semantics: existing documents skip by default, replace
with --overwrite (remove + add; the library API has no content update)."""
def _bundle_with_doc(self, tmp_path):
return write_bundle(tmp_path / "doc.tgx", {
"knowledge/library/doc-1.meta.json": json.dumps(
{"id": "doc-1", "title": "T", "metadata": []}).encode(),
"knowledge/library/doc-1.content": b"hello",
})
def test_existing_document_is_skipped_not_fatal(self, tmp_path):
path = self._bundle_with_doc(tmp_path)
mock_api, mock_config = make_mock_api(
docs=[SimpleNamespace(id="doc-1")],
)
run_import(mock_api, path, overwrite=False)
lib = mock_api.library.return_value
lib.add_document.assert_not_called()
lib.remove_document.assert_not_called()
def test_overwrite_replaces_existing_document(self, tmp_path):
path = self._bundle_with_doc(tmp_path)
mock_api, mock_config = make_mock_api(
docs=[SimpleNamespace(id="doc-1")],
)
run_import(mock_api, path, overwrite=True)
lib = mock_api.library.return_value
lib.remove_document.assert_called_once_with("doc-1")
lib.add_document.assert_called_once()
class TestCollectionDiscovery:
"""Live-verified: implicitly-created collections (raw triple loads) are
queryable but unlisted, so export merges --collection extras and import
registers what it restores."""
def test_export_includes_extra_unregistered_collections(self, tmp_path):
path = tmp_path / "ws.tgx"
mock_api, _ = export_bundle(
path,
collections=[SimpleNamespace(collection="default")],
batches=[[]],
extra_collections=["research"],
)
with tarfile.open(path, "r:gz") as tar:
manifest = json.load(tar.extractfile("manifest.json"))
assert manifest["knowledge"]["collections"] == ["default", "research"]
def test_import_registers_each_restored_collection(self, tmp_path):
path = write_bundle(tmp_path / "kb.tgx", {
"knowledge/research/triples.nq": (
b'<http://ex.com/s> <http://ex.com/p> "v" '
b'<urn:trustgraph:collection:research> .\n'
),
})
mock_api, mock_config = make_mock_api()
run_import(mock_api, path)
mock_api.collection.return_value.update_collection.assert_called_once_with(
"research", name="research",
)
def test_registered_collections_are_not_reregistered(self, tmp_path):
"""update_collection is an upsert that clears omitted fields, so a
collection already in the registry must be left untouched."""
path = write_bundle(tmp_path / "kb.tgx", {
"knowledge/research/triples.nq": (
b'<http://ex.com/s> <http://ex.com/p> "v" '
b'<urn:trustgraph:collection:research> .\n'
),
})
mock_api, mock_config = make_mock_api(
collections=[SimpleNamespace(collection="research")],
)
run_import(mock_api, path)
mock_api.collection.return_value.update_collection.assert_not_called()