mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-06 20:02:11 +02:00
* feat: streaming N-Quads serializer for wire-format triples Groundwork for Phase 2 of #877 (knowledge export). Hand-rolled N-Triples term encoding: rdflib's term.n3() emits Turtle-style forms (numeric shorthand, unescaped newlines) that are invalid in line-oriented N-Quads, so literals are escaped per the ECHAR grammar and IRIs validated for representability. Round-trip tests parse the output back with rdflib's nquads parser and compare term-for-term. * feat: export/import workspace knowledge in .tgx bundles (#877) Phase 2 of the workspace bundle commands: tg-export-workspace now includes the workspace's knowledge by default — per-collection knowledge-graph triples as N-Quads (the collection names the graph, streamed through a tempfile so memory stays flat regardless of knowledge-base size) and the document library (metadata plus content, fetched one document at a time). --config-only skips knowledge on both sides; --triples-limit bounds very large graphs; -f/--flow-id selects the flow the triples services run through. tg-import-workspace streams triples back through the bulk import per collection and recreates library documents (children after parents). Knowledge import is additive, unlike config's skip-existing semantics. Embedding vectors are not carried in bundles: --process re-runs imported documents through the flow, which regenerates extraction output and embeddings; --process-collection targets it. Round-trip covered by unit tests over real archives: export with a mocked Api, re-import, and assert the bulk triples stream and library add calls reproduce the original values (including datatyped literals via the N-Quads path).
This commit is contained in:
Sunny Yang
GitHub
parent
68e816e65c
commit
cfbd5b9079
6 changed files with 1040 additions and 181 deletions
|
|
@ -1,14 +1,14 @@
|
|||
"""
|
||||
Exports a workspace's full configuration state as a portable .tgx bundle
|
||||
(a gzipped tar archive) for backup, migration between deployments, or
|
||||
sharing a pre-configured workspace.
|
||||
Exports a workspace's full state as a portable .tgx bundle (a gzipped tar
|
||||
archive) for backup, migration between deployments, or sharing a
|
||||
pre-configured workspace.
|
||||
|
||||
The bundle is human-readable: a manifest.json plus one pretty-printed JSON
|
||||
file per config key under config/<type>/, so it can be inspected and
|
||||
hand-edited before import. Each entry file embeds its own type and key, so
|
||||
filenames are cosmetic. Knowledge export (triples, documents, embeddings)
|
||||
is not yet included; the manifest records that so future importers can
|
||||
distinguish config-only bundles.
|
||||
The bundle carries the workspace configuration (one pretty-printed JSON
|
||||
file per config key) and, by default, its knowledge: per-collection
|
||||
knowledge-graph triples as N-Quads (the collection names the graph) and
|
||||
the document library (metadata plus content). Pass --config-only to
|
||||
export just the configuration. Embedding vectors are not exported —
|
||||
re-processing imported documents through a flow regenerates them.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
|
@ -17,11 +17,14 @@ import json
|
|||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import time
|
||||
from urllib.parse import quote
|
||||
|
||||
from trustgraph.api import Api
|
||||
|
||||
from . nquads import serialize_nquads
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_token = os.getenv("TRUSTGRAPH_TOKEN", None)
|
||||
default_workspace = os.getenv("TRUSTGRAPH_WORKSPACE", "default")
|
||||
|
|
@ -29,6 +32,10 @@ default_workspace = os.getenv("TRUSTGRAPH_WORKSPACE", "default")
|
|||
MANIFEST_FORMAT = "tgx"
|
||||
MANIFEST_FORMAT_VERSION = 1
|
||||
|
||||
# triples_query_stream is bounded by a limit; exports want "everything", so
|
||||
# default high and let --triples-limit override for truly huge graphs.
|
||||
DEFAULT_TRIPLES_LIMIT = 1_000_000
|
||||
|
||||
|
||||
def _add_bytes(tar, name, data):
|
||||
info = tarfile.TarInfo(name=name)
|
||||
|
|
@ -37,56 +44,176 @@ def _add_bytes(tar, name, data):
|
|||
tar.addfile(info, io.BytesIO(data))
|
||||
|
||||
|
||||
def export_workspace(url, workspace, output, token=None):
|
||||
|
||||
api = Api(url, token=token, workspace=workspace).config()
|
||||
|
||||
config, version = api.all()
|
||||
|
||||
manifest = {
|
||||
"format": MANIFEST_FORMAT,
|
||||
"format_version": MANIFEST_FORMAT_VERSION,
|
||||
"workspace": workspace,
|
||||
"config_version": version,
|
||||
"exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"contents": {"config": True, "knowledge": False},
|
||||
}
|
||||
|
||||
def _export_config(tar, config):
|
||||
"""Write one self-describing JSON file per config key; return count."""
|
||||
count = 0
|
||||
for type_, entries in sorted(config.items()):
|
||||
for key, raw in sorted(entries.items()):
|
||||
|
||||
# Config values are stored as JSON strings; parse so the
|
||||
# bundle is pretty-printed and hand-editable. A value that
|
||||
# isn't valid JSON is preserved verbatim.
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
value = raw
|
||||
|
||||
entry = {"type": type_, "key": key, "value": value}
|
||||
|
||||
# Keys may contain path-unsafe characters; the entry embeds
|
||||
# the real key, so the quoted filename is cosmetic only.
|
||||
name = f"config/{quote(type_, safe='')}/{quote(key, safe='')}.json"
|
||||
|
||||
_add_bytes(
|
||||
tar, name,
|
||||
json.dumps(entry, indent=2).encode("utf-8"),
|
||||
)
|
||||
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def _export_triples(tar, api, flow_id, collections, triples_limit):
|
||||
"""Stream each collection's triples into knowledge/<c>/triples.nq.
|
||||
|
||||
N-Quads are written to a tempfile first (tar members need their size
|
||||
upfront), so memory stays flat regardless of knowledge-base size.
|
||||
Returns {collection: written} and a total skipped count.
|
||||
"""
|
||||
counts = {}
|
||||
skipped_total = 0
|
||||
socket = api.socket()
|
||||
try:
|
||||
flow = socket.flow(flow_id)
|
||||
for c in collections:
|
||||
graph_iri = f"urn:trustgraph:collection:{quote(c, safe='')}"
|
||||
tmp = tempfile.NamedTemporaryFile(
|
||||
"w", encoding="utf-8", suffix=".nq", delete=False,
|
||||
)
|
||||
try:
|
||||
with tmp:
|
||||
written, skipped = serialize_nquads(
|
||||
flow.triples_query_stream(
|
||||
s=None, p=None, o=None,
|
||||
collection=c,
|
||||
limit=triples_limit,
|
||||
batch_size=100,
|
||||
),
|
||||
graph_iri,
|
||||
tmp,
|
||||
)
|
||||
if written:
|
||||
tar.add(
|
||||
tmp.name,
|
||||
arcname=(
|
||||
f"knowledge/{quote(c, safe='')}/triples.nq"
|
||||
),
|
||||
)
|
||||
counts[c] = written
|
||||
skipped_total += skipped
|
||||
finally:
|
||||
os.unlink(tmp.name)
|
||||
finally:
|
||||
socket.close()
|
||||
return counts, skipped_total
|
||||
|
||||
|
||||
def _export_library(tar, api):
|
||||
"""Write each library document's metadata + content; return count."""
|
||||
library = api.library()
|
||||
count = 0
|
||||
for doc in library.get_documents(include_children=True):
|
||||
# Content is fetched one document at a time so memory is bounded
|
||||
# by the largest single document, not the whole library.
|
||||
content = library.get_document_content(doc.id)
|
||||
meta = {
|
||||
"id": doc.id,
|
||||
"time": doc.time.isoformat() if doc.time else None,
|
||||
"kind": doc.kind,
|
||||
"title": doc.title,
|
||||
"comments": doc.comments,
|
||||
"metadata": [
|
||||
{"s": t.s, "p": t.p, "o": t.o} for t in (doc.metadata or [])
|
||||
],
|
||||
"tags": list(doc.tags or []),
|
||||
"parent_id": doc.parent_id or "",
|
||||
"document_type": getattr(doc, "document_type", "") or "",
|
||||
}
|
||||
base = f"knowledge/library/{quote(doc.id, safe='')}"
|
||||
_add_bytes(
|
||||
tar, f"{base}.meta.json",
|
||||
json.dumps(meta, indent=2).encode("utf-8"),
|
||||
)
|
||||
_add_bytes(tar, f"{base}.content", content or b"")
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
def export_workspace(
|
||||
url, workspace, output, token=None, config_only=False,
|
||||
flow_id="default", triples_limit=DEFAULT_TRIPLES_LIMIT,
|
||||
extra_collections=(),
|
||||
):
|
||||
|
||||
api = Api(url, token=token, workspace=workspace)
|
||||
|
||||
config, version = api.config().all()
|
||||
|
||||
# Collection discovery is registry-based: collections created implicitly
|
||||
# by raw triple loads (e.g. tg-load-knowledge) are queryable but not
|
||||
# listed, so they would silently drop out of the bundle. --collection
|
||||
# names them explicitly; the enumeration is printed so what's included
|
||||
# is never a guess.
|
||||
collections = []
|
||||
if not config_only:
|
||||
registered = [c.collection for c in api.collection().list_collections()]
|
||||
collections = sorted(set(registered) | set(extra_collections))
|
||||
print(f"Exporting collections: {', '.join(collections)}", flush=True)
|
||||
|
||||
with tarfile.open(output, "w:gz") as tar:
|
||||
|
||||
config_count = _export_config(tar, config)
|
||||
|
||||
triple_counts = {}
|
||||
skipped = 0
|
||||
doc_count = 0
|
||||
if not config_only:
|
||||
triple_counts, skipped = _export_triples(
|
||||
tar, api, flow_id, collections, triples_limit,
|
||||
)
|
||||
doc_count = _export_library(tar, api)
|
||||
|
||||
manifest = {
|
||||
"format": MANIFEST_FORMAT,
|
||||
"format_version": MANIFEST_FORMAT_VERSION,
|
||||
"workspace": workspace,
|
||||
"config_version": version,
|
||||
"exported_at": time.strftime(
|
||||
"%Y-%m-%dT%H:%M:%SZ", time.gmtime(),
|
||||
),
|
||||
"contents": {"config": True, "knowledge": not config_only},
|
||||
}
|
||||
if not config_only:
|
||||
manifest["knowledge"] = {
|
||||
"collections": collections,
|
||||
"documents": doc_count,
|
||||
"triples": triple_counts,
|
||||
}
|
||||
|
||||
_add_bytes(
|
||||
tar, "manifest.json",
|
||||
json.dumps(manifest, indent=2).encode("utf-8"),
|
||||
)
|
||||
|
||||
for type_, entries in sorted(config.items()):
|
||||
for key, raw in sorted(entries.items()):
|
||||
|
||||
# Config values are stored as JSON strings; parse so the
|
||||
# bundle is pretty-printed and hand-editable. A value that
|
||||
# isn't valid JSON is preserved verbatim.
|
||||
try:
|
||||
value = json.loads(raw)
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
value = raw
|
||||
|
||||
entry = {"type": type_, "key": key, "value": value}
|
||||
|
||||
# Keys may contain path-unsafe characters; the entry embeds
|
||||
# the real key, so the quoted filename is cosmetic only.
|
||||
name = f"config/{quote(type_, safe='')}/{quote(key, safe='')}.json"
|
||||
|
||||
_add_bytes(
|
||||
tar, name,
|
||||
json.dumps(entry, indent=2).encode("utf-8"),
|
||||
)
|
||||
|
||||
count += 1
|
||||
|
||||
print(f"Exported {count} config item(s) from workspace "
|
||||
f"'{workspace}' to {output}", flush=True)
|
||||
summary = f"Exported {config_count} config item(s)"
|
||||
if not config_only:
|
||||
summary += (
|
||||
f", {sum(triple_counts.values())} triple(s) across "
|
||||
f"{len(triple_counts)} collection(s), {doc_count} document(s)"
|
||||
)
|
||||
if skipped:
|
||||
summary += f" ({skipped} triple(s) not representable, skipped)"
|
||||
print(f"{summary} from workspace '{workspace}' to {output}", flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -114,12 +241,41 @@ def main():
|
|||
help=f'Workspace to export (default: {default_workspace})',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-c', '--collection',
|
||||
action='append',
|
||||
default=[],
|
||||
help='Additionally export this collection even if it is not '
|
||||
'registered in collection management (repeatable)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-o', '--output',
|
||||
required=True,
|
||||
help='Output bundle path, e.g. workspace-default.tgx',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-f', '--flow-id',
|
||||
default="default",
|
||||
help='Flow to query triples through (default: default)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--config-only',
|
||||
action='store_true',
|
||||
help='Export only the configuration, skipping knowledge '
|
||||
'(triples and library documents)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--triples-limit',
|
||||
type=int,
|
||||
default=DEFAULT_TRIPLES_LIMIT,
|
||||
help='Maximum triples to export per collection '
|
||||
f'(default: {DEFAULT_TRIPLES_LIMIT})',
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
|
|
@ -129,6 +285,10 @@ def main():
|
|||
workspace=args.workspace,
|
||||
output=args.output,
|
||||
token=args.token,
|
||||
config_only=args.config_only,
|
||||
flow_id=args.flow_id,
|
||||
triples_limit=args.triples_limit,
|
||||
extra_collections=args.collection,
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
|
|
|
|||
|
|
@ -3,10 +3,16 @@ Imports a workspace bundle (.tgx, produced by tg-export-workspace) into a
|
|||
TrustGraph deployment. The target workspace defaults to the name recorded
|
||||
in the bundle's manifest and can be renamed with --workspace.
|
||||
|
||||
By default existing (type, key) entries in the target workspace are left
|
||||
untouched and only missing keys are added, matching WorkspaceInit's
|
||||
re-run behaviour; pass --overwrite to replace every imported key. Use
|
||||
--dry-run to show what would be written without changing anything.
|
||||
Configuration import follows WorkspaceInit's re-run behaviour: existing
|
||||
(type, key) entries are left untouched and only missing keys are added;
|
||||
pass --overwrite to replace every imported key. Knowledge import (triples
|
||||
and library documents) is additive — triples are streamed into the target
|
||||
collection and documents are added to the library; re-importing the same
|
||||
bundle twice will duplicate knowledge, not merge it. Use --dry-run to
|
||||
show what would be written without changing anything, --config-only to
|
||||
skip a bundle's knowledge, and --process to re-run imported documents
|
||||
through a flow (which regenerates embeddings, so bundles don't carry
|
||||
vectors).
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
|
@ -14,9 +20,12 @@ import json
|
|||
import os
|
||||
import sys
|
||||
import tarfile
|
||||
import uuid
|
||||
from urllib.parse import unquote
|
||||
|
||||
from trustgraph.api import Api
|
||||
from trustgraph.api.types import ConfigValue
|
||||
from trustgraph.api.types import ConfigValue, Triple
|
||||
from trustgraph.cli.nquads import parse_nquads
|
||||
|
||||
default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/')
|
||||
default_token = os.getenv("TRUSTGRAPH_TOKEN", None)
|
||||
|
|
@ -26,10 +35,21 @@ SUPPORTED_FORMAT_VERSION = 1
|
|||
|
||||
|
||||
def _read_bundle(path):
|
||||
"""Read manifest and config entries from a .tgx bundle."""
|
||||
"""Read manifest, config entries, triples and documents from a .tgx.
|
||||
|
||||
Returns (manifest, config_entries, triples_by_collection, documents)
|
||||
where triples_by_collection maps collection -> list[Triple] and
|
||||
documents is a list of {meta: dict, content: bytes}.
|
||||
"""
|
||||
|
||||
manifest = None
|
||||
entries = []
|
||||
config_entries = []
|
||||
triples = {}
|
||||
doc_meta = {}
|
||||
doc_content = {}
|
||||
|
||||
def member_id(name, prefix, suffix):
|
||||
return unquote(name[len(prefix):-len(suffix)])
|
||||
|
||||
with tarfile.open(path, "r:gz") as tar:
|
||||
for member in tar.getmembers():
|
||||
|
|
@ -39,11 +59,28 @@ def _read_bundle(path):
|
|||
if f is None:
|
||||
continue
|
||||
data = f.read()
|
||||
if member.name == "manifest.json":
|
||||
name = member.name
|
||||
|
||||
if name == "manifest.json":
|
||||
manifest = json.loads(data)
|
||||
elif member.name.startswith("config/") and \
|
||||
member.name.endswith(".json"):
|
||||
entries.append(json.loads(data))
|
||||
|
||||
elif name.startswith("config/") and name.endswith(".json"):
|
||||
config_entries.append(json.loads(data))
|
||||
|
||||
elif name.startswith("knowledge/library/") and \
|
||||
name.endswith(".meta.json"):
|
||||
doc_id = member_id(name, "knowledge/library/", ".meta.json")
|
||||
doc_meta[doc_id] = json.loads(data)
|
||||
|
||||
elif name.startswith("knowledge/library/") and \
|
||||
name.endswith(".content"):
|
||||
doc_id = member_id(name, "knowledge/library/", ".content")
|
||||
doc_content[doc_id] = data
|
||||
|
||||
elif name.startswith("knowledge/") and \
|
||||
name.endswith("/triples.nq"):
|
||||
collection = member_id(name, "knowledge/", "/triples.nq")
|
||||
triples[collection] = parse_nquads(data)
|
||||
|
||||
if manifest is None:
|
||||
raise RuntimeError("not a workspace bundle: manifest.json missing")
|
||||
|
|
@ -60,27 +97,18 @@ def _read_bundle(path):
|
|||
"upgrade trustgraph-cli"
|
||||
)
|
||||
|
||||
return manifest, entries
|
||||
documents = [
|
||||
{"meta": meta, "content": doc_content.get(doc_id, b"")}
|
||||
for doc_id, meta in doc_meta.items()
|
||||
]
|
||||
|
||||
return manifest, config_entries, triples, documents
|
||||
|
||||
|
||||
def import_workspace(
|
||||
url, input, workspace=None, overwrite=False, config_only=False,
|
||||
dry_run=False, token=None,
|
||||
):
|
||||
def _import_config(api, entries, overwrite, dry_run):
|
||||
"""Import config entries; returns (imported, skipped) counts."""
|
||||
|
||||
manifest, entries = _read_bundle(input)
|
||||
|
||||
# Knowledge import (triples, documents, embeddings) is not implemented
|
||||
# yet; refuse to silently drop it from a bundle that carries it.
|
||||
if manifest.get("contents", {}).get("knowledge") and not config_only:
|
||||
raise RuntimeError(
|
||||
"bundle contains knowledge data, which this tool cannot import "
|
||||
"yet; re-run with --config-only to import just the configuration"
|
||||
)
|
||||
|
||||
target = workspace or manifest.get("workspace") or "default"
|
||||
|
||||
api = Api(url, token=token, workspace=target).config()
|
||||
config = api.config()
|
||||
|
||||
# Mirror WorkspaceInit's re-run behaviour: without --overwrite, keys
|
||||
# already present in the target workspace are skipped (per key, not per
|
||||
|
|
@ -88,7 +116,7 @@ def import_workspace(
|
|||
existing = {}
|
||||
if not overwrite:
|
||||
for type_ in sorted({e["type"] for e in entries}):
|
||||
existing[type_] = set(api.list(type_))
|
||||
existing[type_] = set(config.list(type_))
|
||||
|
||||
values = []
|
||||
skipped = 0
|
||||
|
|
@ -106,16 +134,191 @@ def import_workspace(
|
|||
if dry_run:
|
||||
for v in values:
|
||||
print(f"would import {v.type}/{v.key}", flush=True)
|
||||
print(f"Dry run: {len(values)} item(s) would be imported into "
|
||||
f"workspace '{target}', {skipped} skipped as existing",
|
||||
flush=True)
|
||||
return
|
||||
elif values:
|
||||
config.put(values)
|
||||
|
||||
if values:
|
||||
api.put(values)
|
||||
return len(values), skipped
|
||||
|
||||
print(f"Imported {len(values)} config item(s) into workspace "
|
||||
f"'{target}', {skipped} skipped as existing", flush=True)
|
||||
|
||||
def _import_triples(api, flow_id, triples_by_collection, dry_run):
|
||||
"""Stream each collection's triples into the flow; returns count."""
|
||||
|
||||
# Collections restored by the bulk triples path (unlike document
|
||||
# processing) are not auto-registered, and an unregistered collection
|
||||
# would silently drop out of a future export. Register only the missing
|
||||
# ones: update_collection is an upsert whose omitted fields clear the
|
||||
# description/tags that _import_config may have just restored.
|
||||
# Best-effort — a registry hiccup shouldn't fail a completed triple
|
||||
# import (tg-set-collection is the manual remedy).
|
||||
registered = None
|
||||
if triples_by_collection and not dry_run:
|
||||
try:
|
||||
registered = {
|
||||
c.collection for c in api.collection().list_collections()
|
||||
}
|
||||
except Exception as e:
|
||||
print(
|
||||
f"warning: could not list collections for registration: "
|
||||
f"{e} — register manually with tg-set-collection",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
total = 0
|
||||
for collection, triples in sorted(triples_by_collection.items()):
|
||||
if dry_run:
|
||||
print(
|
||||
f"would import {len(triples)} triple(s) into "
|
||||
f"collection '{collection}'",
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
api.bulk().import_triples(
|
||||
flow_id,
|
||||
triples,
|
||||
metadata={
|
||||
"id": f"workspace-import-{uuid.uuid4()}",
|
||||
"metadata": [],
|
||||
"collection": collection,
|
||||
},
|
||||
)
|
||||
if registered is not None and collection not in registered:
|
||||
try:
|
||||
api.collection().update_collection(
|
||||
collection, name=collection,
|
||||
)
|
||||
except Exception as e:
|
||||
print(
|
||||
f"warning: could not register collection "
|
||||
f"'{collection}': {e} — register manually with "
|
||||
f"tg-set-collection",
|
||||
flush=True,
|
||||
)
|
||||
total += len(triples)
|
||||
return total
|
||||
|
||||
|
||||
def _import_documents(
|
||||
api, documents, flow_id, process, process_collection, overwrite,
|
||||
dry_run,
|
||||
):
|
||||
"""Add library documents back (children after parents).
|
||||
|
||||
Mirrors the config semantics: documents already present in the target
|
||||
workspace are skipped unless --overwrite, which replaces them (the
|
||||
library API has no in-place content update, so replace = remove + add).
|
||||
Returns (imported, skipped).
|
||||
"""
|
||||
|
||||
library = api.library()
|
||||
|
||||
existing = set()
|
||||
if documents:
|
||||
existing = {d.id for d in library.get_documents(include_children=True)}
|
||||
|
||||
# Parents must exist before their children.
|
||||
ordered = sorted(
|
||||
documents, key=lambda d: bool(d["meta"].get("parent_id")),
|
||||
)
|
||||
|
||||
count = 0
|
||||
doc_skipped = 0
|
||||
for doc in ordered:
|
||||
meta = doc["meta"]
|
||||
|
||||
if meta["id"] in existing and not overwrite:
|
||||
doc_skipped += 1
|
||||
if dry_run:
|
||||
print(f"would skip existing document {meta['id']}", flush=True)
|
||||
continue
|
||||
|
||||
if dry_run:
|
||||
print(f"would import document {meta['id']}", flush=True)
|
||||
count += 1
|
||||
continue
|
||||
|
||||
if meta["id"] in existing:
|
||||
library.remove_document(meta["id"])
|
||||
|
||||
metadata = [
|
||||
Triple(s=t["s"], p=t["p"], o=t["o"])
|
||||
for t in meta.get("metadata", [])
|
||||
]
|
||||
if meta.get("parent_id"):
|
||||
library.add_child_document(
|
||||
document=doc["content"],
|
||||
id=meta["id"],
|
||||
parent_id=meta["parent_id"],
|
||||
title=meta.get("title", ""),
|
||||
comments=meta.get("comments", ""),
|
||||
kind=meta.get("kind", "text/plain"),
|
||||
tags=meta.get("tags", []),
|
||||
metadata=metadata,
|
||||
)
|
||||
else:
|
||||
library.add_document(
|
||||
document=doc["content"],
|
||||
id=meta["id"],
|
||||
metadata=metadata,
|
||||
title=meta.get("title", ""),
|
||||
comments=meta.get("comments", ""),
|
||||
kind=meta.get("kind", "text/plain"),
|
||||
tags=meta.get("tags", []),
|
||||
)
|
||||
|
||||
# Re-processing regenerates extraction output and embeddings for
|
||||
# the imported content (bundles carry no vectors).
|
||||
if process:
|
||||
library.start_processing(
|
||||
id=f"proc-{uuid.uuid4()}",
|
||||
document_id=meta["id"],
|
||||
flow=flow_id,
|
||||
collection=process_collection,
|
||||
tags=meta.get("tags", []),
|
||||
)
|
||||
|
||||
count += 1
|
||||
return count, doc_skipped
|
||||
|
||||
|
||||
def import_workspace(
|
||||
url, input, workspace=None, overwrite=False, config_only=False,
|
||||
flow_id="default", process=False, process_collection="default",
|
||||
dry_run=False, token=None,
|
||||
):
|
||||
|
||||
manifest, config_entries, triples, documents = _read_bundle(input)
|
||||
|
||||
target = workspace or manifest.get("workspace") or "default"
|
||||
|
||||
api = Api(url, token=token, workspace=target)
|
||||
|
||||
imported, skipped = _import_config(
|
||||
api, config_entries, overwrite, dry_run,
|
||||
)
|
||||
|
||||
triple_count = 0
|
||||
doc_count = 0
|
||||
has_knowledge = bool(triples or documents)
|
||||
if has_knowledge and not config_only:
|
||||
triple_count = _import_triples(api, flow_id, triples, dry_run)
|
||||
doc_count, doc_skipped = _import_documents(
|
||||
api, documents, flow_id, process, process_collection, overwrite,
|
||||
dry_run,
|
||||
)
|
||||
|
||||
verb = "Dry run:" if dry_run else "Imported"
|
||||
summary = (
|
||||
f"{verb} {imported} config item(s) into workspace '{target}', "
|
||||
f"{skipped} skipped as existing"
|
||||
)
|
||||
if has_knowledge and not config_only:
|
||||
summary += (
|
||||
f"; {triple_count} triple(s), {doc_count} document(s)"
|
||||
f" ({doc_skipped} skipped as existing)"
|
||||
)
|
||||
elif has_knowledge:
|
||||
summary += "; knowledge skipped (--config-only)"
|
||||
print(summary, flush=True)
|
||||
|
||||
|
||||
def main():
|
||||
|
|
@ -150,11 +353,18 @@ def main():
|
|||
'bundle manifest)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'-f', '--flow-id',
|
||||
default="default",
|
||||
help='Flow to import triples through and process documents with '
|
||||
'(default: default)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--overwrite',
|
||||
action='store_true',
|
||||
help='Replace existing keys in the target workspace (default: '
|
||||
'keep existing keys and only add missing ones)',
|
||||
help='Replace existing config keys in the target workspace '
|
||||
'(default: keep existing keys and only add missing ones)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
|
|
@ -164,6 +374,19 @@ def main():
|
|||
'in the bundle',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--process',
|
||||
action='store_true',
|
||||
help='Re-process imported documents through the flow after import '
|
||||
'(regenerates extraction output and embeddings)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--process-collection',
|
||||
default="default",
|
||||
help='Collection that --process targets (default: default)',
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
'--dry-run',
|
||||
action='store_true',
|
||||
|
|
@ -180,6 +403,9 @@ def main():
|
|||
workspace=args.workspace,
|
||||
overwrite=args.overwrite,
|
||||
config_only=args.config_only,
|
||||
flow_id=args.flow_id,
|
||||
process=args.process,
|
||||
process_collection=args.process_collection,
|
||||
dry_run=args.dry_run,
|
||||
token=args.token,
|
||||
)
|
||||
|
|
|
|||
137
trustgraph-cli/trustgraph/cli/nquads.py
Normal file
137
trustgraph-cli/trustgraph/cli/nquads.py
Normal file
|
|
@ -0,0 +1,137 @@
|
|||
"""
|
||||
N-Quads serialization and parsing for workspace knowledge bundles: the
|
||||
wire-format triples yielded by triples_query_stream go out one line per
|
||||
triple (so an export never holds a whole graph in memory), and bundle
|
||||
members come back as api Triple values. Term encoding is hand-rolled to
|
||||
the N-Triples grammar: rdflib's term.n3() emits Turtle-style forms
|
||||
(numeric shorthand, unescaped newlines) that are not valid in
|
||||
line-oriented N-Quads.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
import rdflib
|
||||
|
||||
from trustgraph.schema import IRI, LITERAL
|
||||
from trustgraph.api.types import Triple
|
||||
|
||||
# RDF-star quoted triples have no standard N-Quads encoding; they are
|
||||
# skipped with a count so callers can surface the omission.
|
||||
|
||||
# N-Triples string-literal escapes (ECHAR): backslash first, then the rest.
|
||||
_ESCAPES = [
|
||||
("\\", "\\\\"),
|
||||
('"', '\\"'),
|
||||
("\n", "\\n"),
|
||||
("\r", "\\r"),
|
||||
("\t", "\\t"),
|
||||
]
|
||||
|
||||
# Characters the IRIREF production cannot carry: controls/space plus the
|
||||
# explicitly forbidden set. One compiled scan keeps this off the profile
|
||||
# for large exports (it runs per term).
|
||||
_BAD_IRI = re.compile(r'[\x00-\x20<>"{}|^\x60]')
|
||||
|
||||
|
||||
def _escape_literal(value):
|
||||
for raw, esc in _ESCAPES:
|
||||
value = value.replace(raw, esc)
|
||||
return value
|
||||
|
||||
|
||||
def _encode_iri(iri):
|
||||
"""<iri>, or None for values the grammar cannot carry."""
|
||||
if not iri or _BAD_IRI.search(iri):
|
||||
return None
|
||||
return f"<{iri}>"
|
||||
|
||||
|
||||
def encode_term(term, is_object=False):
|
||||
"""Encode one wire-format term dict for an N-Quads line.
|
||||
|
||||
:param is_object: literals are only valid in object position;
|
||||
subjects and predicates must be IRIs (bnodes never appear on
|
||||
the wire).
|
||||
:returns: encoded string, or None when the term can't be represented.
|
||||
"""
|
||||
if term is None:
|
||||
return None
|
||||
|
||||
t = term.get("t", "")
|
||||
|
||||
if t == IRI:
|
||||
return _encode_iri(term.get("i", ""))
|
||||
|
||||
if t == LITERAL and is_object:
|
||||
value = _escape_literal(term.get("v", ""))
|
||||
language = term.get("l")
|
||||
datatype = term.get("d")
|
||||
if language:
|
||||
return f'"{value}"@{language}'
|
||||
if datatype:
|
||||
dt = _encode_iri(datatype)
|
||||
if dt is None:
|
||||
return None
|
||||
return f'"{value}"^^{dt}'
|
||||
return f'"{value}"'
|
||||
|
||||
# literals outside object position, RDF-star, unknown types
|
||||
return None
|
||||
|
||||
|
||||
def triple_to_nquad(triple, graph_encoded):
|
||||
"""One wire-format triple dict -> an N-Quads line, or None to skip.
|
||||
|
||||
:param triple: {"s": term, "p": term, "o": term} wire dict
|
||||
:param graph_encoded: pre-encoded <graph-iri> string
|
||||
"""
|
||||
s = encode_term(triple.get("s"))
|
||||
p = encode_term(triple.get("p"))
|
||||
o = encode_term(triple.get("o"), is_object=True)
|
||||
if s is None or p is None or o is None:
|
||||
return None
|
||||
return f"{s} {p} {o} {graph_encoded} .\n"
|
||||
|
||||
|
||||
def serialize_nquads(batches, graph_iri, out):
|
||||
"""Write wire-format triple batches to a text file-like as N-Quads.
|
||||
|
||||
:param batches: iterable of lists of wire triple dicts
|
||||
(e.g. triples_query_stream output)
|
||||
:param graph_iri: graph name for every quad (str)
|
||||
:param out: text-mode file-like with .write()
|
||||
:returns: (written, skipped) counts
|
||||
"""
|
||||
g = _encode_iri(graph_iri)
|
||||
if g is None:
|
||||
raise ValueError(f"graph IRI not representable in N-Quads: {graph_iri!r}")
|
||||
written = 0
|
||||
skipped = 0
|
||||
for batch in batches:
|
||||
for t in batch:
|
||||
line = triple_to_nquad(t, g)
|
||||
if line is None:
|
||||
skipped += 1
|
||||
else:
|
||||
out.write(line)
|
||||
written += 1
|
||||
return written, skipped
|
||||
|
||||
|
||||
def parse_nquads(data):
|
||||
"""Parse N-Quads bytes back into api Triple values.
|
||||
|
||||
Terms are stringified with str(), the same convention tg-load-knowledge
|
||||
uses, so values survive the store round trip unchanged. The whole
|
||||
member is materialized in memory (bundles are bounded by
|
||||
--triples-limit at export); line-streaming is a possible follow-up.
|
||||
|
||||
:param data: N-Quads bytes (one bundle member)
|
||||
:returns: list of Triple
|
||||
"""
|
||||
ds = rdflib.Dataset()
|
||||
ds.parse(data=data.decode("utf-8"), format="nquads")
|
||||
return [
|
||||
Triple(s=str(s), p=str(p), o=str(o))
|
||||
for s, p, o, _g in ds.quads((None, None, None, None))
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue