From 76c4763b9b8a07d5656821d83d4ea35010eb0a9e Mon Sep 17 00:00:00 2001 From: Sunny Yang Date: Fri, 3 Jul 2026 08:16:39 -0600 Subject: [PATCH] feat: add tg-export-workspace / tg-import-workspace bundle commands (#877) (#1019) Phase 1 (config only): export a workspace's full configuration as a portable .tgx bundle (gzipped tar with manifest.json and one pretty-printed, self-describing JSON file per config key under config//), and import it into another deployment or workspace. Import defaults to WorkspaceInit's re-run semantics (existing keys kept, missing keys added; --overwrite replaces), supports --workspace rename, --dry-run, and --config-only, and refuses to silently drop knowledge data from future Phase-2 bundles it cannot import yet. --- .../test_workspace_bundle_commands.py | 224 ++++++++++++++++++ trustgraph-cli/pyproject.toml | 2 + .../trustgraph/cli/export_workspace.py | 141 +++++++++++ .../trustgraph/cli/import_workspace.py | 194 +++++++++++++++ 4 files changed, 561 insertions(+) create mode 100644 tests/unit/test_cli/test_workspace_bundle_commands.py create mode 100644 trustgraph-cli/trustgraph/cli/export_workspace.py create mode 100644 trustgraph-cli/trustgraph/cli/import_workspace.py diff --git a/tests/unit/test_cli/test_workspace_bundle_commands.py b/tests/unit/test_cli/test_workspace_bundle_commands.py new file mode 100644 index 00000000..e45c0ef4 --- /dev/null +++ b/tests/unit/test_cli/test_workspace_bundle_commands.py @@ -0,0 +1,224 @@ +""" +Tests for tg-export-workspace / tg-import-workspace (.tgx bundle commands). + +The Api class is mocked in each command module's namespace (same pattern as +test_config_commands.py); bundles are written to and read from tmp_path so +the archive format itself is exercised end-to-end. +""" + +import json +import tarfile +from unittest.mock import Mock, patch + +import pytest + +from trustgraph.api.types import ConfigValue +from trustgraph.cli.export_workspace import export_workspace +from trustgraph.cli.import_workspace import import_workspace + +SAMPLE_CONFIG = { + "prompt": { + "extract-concepts": json.dumps({"template": "Extract {{q}}"}), + "answer": json.dumps({"template": "Answer {{q}}"}), + }, + "tool": { + "web-search": json.dumps({"name": "web-search", "kind": "http"}), + }, +} + + +def make_mock_api(): + mock_config = Mock() + mock_api = Mock() + mock_api.config.return_value = mock_config + return mock_api, mock_config + + +@pytest.fixture +def bundle(tmp_path): + """Export SAMPLE_CONFIG to a real .tgx and yield its path.""" + path = tmp_path / "ws.tgx" + with patch("trustgraph.cli.export_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + mock_config.all.return_value = (SAMPLE_CONFIG, "v42") + export_workspace( + url="http://api/", workspace="source-ws", output=str(path), + ) + return path + + +class TestExportWorkspace: + + def test_bundle_contains_manifest_and_per_key_entries(self, bundle): + with tarfile.open(bundle, "r:gz") as tar: + names = tar.getnames() + manifest = json.load(tar.extractfile("manifest.json")) + + assert manifest["format"] == "tgx" + assert manifest["workspace"] == "source-ws" + assert manifest["config_version"] == "v42" + assert manifest["contents"] == {"config": True, "knowledge": False} + + assert "config/prompt/extract-concepts.json" in names + assert "config/prompt/answer.json" in names + assert "config/tool/web-search.json" in names + + def test_entries_are_parsed_and_self_describing(self, bundle): + with tarfile.open(bundle, "r:gz") as tar: + entry = json.load( + tar.extractfile("config/prompt/extract-concepts.json") + ) + # Values are pretty-printed objects, not double-encoded strings, + # and each entry embeds its own type/key (filenames are cosmetic). + assert entry == { + "type": "prompt", + "key": "extract-concepts", + "value": {"template": "Extract {{q}}"}, + } + + def test_path_unsafe_keys_are_quoted_in_filenames(self, tmp_path): + path = tmp_path / "ws.tgx" + with patch("trustgraph.cli.export_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + mock_config.all.return_value = ( + {"prompt": {"a/b": json.dumps({"x": 1})}}, "v1", + ) + export_workspace( + url="http://api/", workspace="ws", output=str(path), + ) + with tarfile.open(path, "r:gz") as tar: + names = tar.getnames() + entry = json.load(tar.extractfile("config/prompt/a%2Fb.json")) + assert "config/prompt/a%2Fb.json" in names + assert entry["key"] == "a/b" + + +class TestImportWorkspace: + + def test_roundtrip_puts_all_values_with_overwrite(self, bundle): + with patch("trustgraph.cli.import_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + import_workspace( + url="http://api/", input=str(bundle), overwrite=True, + ) + + # Target workspace defaults to the manifest's workspace. + api_cls.assert_called_once_with( + "http://api/", token=None, workspace="source-ws", + ) + values = mock_config.put.call_args.args[0] + assert sorted((v.type, v.key) for v in values) == [ + ("prompt", "answer"), + ("prompt", "extract-concepts"), + ("tool", "web-search"), + ] + # Values are re-serialized to JSON strings, as config-svc stores. + by_key = {(v.type, v.key): v for v in values} + assert json.loads(by_key[("prompt", "answer")].value) == { + "template": "Answer {{q}}", + } + assert all(isinstance(v, ConfigValue) for v in values) + + def test_workspace_flag_renames_target(self, bundle): + with patch("trustgraph.cli.import_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + import_workspace( + url="http://api/", input=str(bundle), workspace="staging", + overwrite=True, + ) + api_cls.assert_called_once_with( + "http://api/", token=None, workspace="staging", + ) + + def test_default_skips_existing_keys(self, bundle): + """WorkspaceInit re-run semantics: only missing keys are written.""" + with patch("trustgraph.cli.import_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + mock_config.list.side_effect = lambda t: { + "prompt": ["extract-concepts"], + "tool": [], + }[t] + import_workspace(url="http://api/", input=str(bundle)) + + values = mock_config.put.call_args.args[0] + assert sorted((v.type, v.key) for v in values) == [ + ("prompt", "answer"), + ("tool", "web-search"), + ] + + def test_dry_run_writes_nothing(self, bundle, capsys): + with patch("trustgraph.cli.import_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + import_workspace( + url="http://api/", input=str(bundle), overwrite=True, + dry_run=True, + ) + mock_config.put.assert_not_called() + out = capsys.readouterr().out + assert "would import prompt/extract-concepts" in out + assert "3 item(s) would be imported" in out + + def test_rejects_bundle_without_manifest(self, tmp_path): + path = tmp_path / "bad.tgx" + with tarfile.open(path, "w:gz"): + pass + with patch("trustgraph.cli.import_workspace.Api"): + with pytest.raises(RuntimeError, match="manifest.json missing"): + import_workspace(url="http://api/", input=str(path)) + + def test_rejects_newer_format_version(self, tmp_path): + import io + path = tmp_path / "future.tgx" + manifest = json.dumps({ + "format": "tgx", "format_version": 99, "workspace": "w", + "contents": {"config": True, "knowledge": False}, + }).encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo("manifest.json") + info.size = len(manifest) + tar.addfile(info, io.BytesIO(manifest)) + with patch("trustgraph.cli.import_workspace.Api"): + with pytest.raises(RuntimeError, match="newer than this tool"): + import_workspace(url="http://api/", input=str(path)) + + def test_refuses_knowledge_bundle_without_config_only(self, tmp_path): + import io + path = tmp_path / "knowledge.tgx" + manifest = json.dumps({ + "format": "tgx", "format_version": 1, "workspace": "w", + "contents": {"config": True, "knowledge": True}, + }).encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo("manifest.json") + info.size = len(manifest) + tar.addfile(info, io.BytesIO(manifest)) + with patch("trustgraph.cli.import_workspace.Api"): + with pytest.raises(RuntimeError, match="--config-only"): + import_workspace(url="http://api/", input=str(path)) + + def test_config_only_flag_allows_knowledge_bundle(self, tmp_path): + import io + path = tmp_path / "knowledge.tgx" + manifest = json.dumps({ + "format": "tgx", "format_version": 1, "workspace": "w", + "contents": {"config": True, "knowledge": True}, + }).encode() + with tarfile.open(path, "w:gz") as tar: + info = tarfile.TarInfo("manifest.json") + info.size = len(manifest) + tar.addfile(info, io.BytesIO(manifest)) + with patch("trustgraph.cli.import_workspace.Api") as api_cls: + mock_api, mock_config = make_mock_api() + api_cls.return_value = mock_api + import_workspace( + url="http://api/", input=str(path), config_only=True, + overwrite=True, + ) + # No config entries in this bundle; nothing written, no error. + mock_config.put.assert_not_called() diff --git a/trustgraph-cli/pyproject.toml b/trustgraph-cli/pyproject.toml index 193ee1cd..a89be8ca 100644 --- a/trustgraph-cli/pyproject.toml +++ b/trustgraph-cli/pyproject.toml @@ -116,6 +116,8 @@ tg-put-config-item = "trustgraph.cli.put_config_item:main" tg-delete-config-item = "trustgraph.cli.delete_config_item:main" tg-export-workspace-config = "trustgraph.cli.export_workspace_config:main" tg-import-workspace-config = "trustgraph.cli.import_workspace_config:main" +tg-export-workspace = "trustgraph.cli.export_workspace:main" +tg-import-workspace = "trustgraph.cli.import_workspace:main" tg-list-collections = "trustgraph.cli.list_collections:main" tg-set-collection = "trustgraph.cli.set_collection:main" tg-delete-collection = "trustgraph.cli.delete_collection:main" diff --git a/trustgraph-cli/trustgraph/cli/export_workspace.py b/trustgraph-cli/trustgraph/cli/export_workspace.py new file mode 100644 index 00000000..41d3403d --- /dev/null +++ b/trustgraph-cli/trustgraph/cli/export_workspace.py @@ -0,0 +1,141 @@ +""" +Exports a workspace's full configuration state as a portable .tgx bundle +(a gzipped tar archive) for backup, migration between deployments, or +sharing a pre-configured workspace. + +The bundle is human-readable: a manifest.json plus one pretty-printed JSON +file per config key under config//, so it can be inspected and +hand-edited before import. Each entry file embeds its own type and key, so +filenames are cosmetic. Knowledge export (triples, documents, embeddings) +is not yet included; the manifest records that so future importers can +distinguish config-only bundles. +""" + +import argparse +import io +import json +import os +import sys +import tarfile +import time +from urllib.parse import quote + +from trustgraph.api import Api + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') +default_token = os.getenv("TRUSTGRAPH_TOKEN", None) +default_workspace = os.getenv("TRUSTGRAPH_WORKSPACE", "default") + +MANIFEST_FORMAT = "tgx" +MANIFEST_FORMAT_VERSION = 1 + + +def _add_bytes(tar, name, data): + info = tarfile.TarInfo(name=name) + info.size = len(data) + info.mtime = int(time.time()) + tar.addfile(info, io.BytesIO(data)) + + +def export_workspace(url, workspace, output, token=None): + + api = Api(url, token=token, workspace=workspace).config() + + config, version = api.all() + + manifest = { + "format": MANIFEST_FORMAT, + "format_version": MANIFEST_FORMAT_VERSION, + "workspace": workspace, + "config_version": version, + "exported_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "contents": {"config": True, "knowledge": False}, + } + + count = 0 + + with tarfile.open(output, "w:gz") as tar: + + _add_bytes( + tar, "manifest.json", + json.dumps(manifest, indent=2).encode("utf-8"), + ) + + for type_, entries in sorted(config.items()): + for key, raw in sorted(entries.items()): + + # Config values are stored as JSON strings; parse so the + # bundle is pretty-printed and hand-editable. A value that + # isn't valid JSON is preserved verbatim. + try: + value = json.loads(raw) + except (TypeError, json.JSONDecodeError): + value = raw + + entry = {"type": type_, "key": key, "value": value} + + # Keys may contain path-unsafe characters; the entry embeds + # the real key, so the quoted filename is cosmetic only. + name = f"config/{quote(type_, safe='')}/{quote(key, safe='')}.json" + + _add_bytes( + tar, name, + json.dumps(entry, indent=2).encode("utf-8"), + ) + + count += 1 + + print(f"Exported {count} config item(s) from workspace " + f"'{workspace}' to {output}", flush=True) + + +def main(): + + parser = argparse.ArgumentParser( + prog='tg-export-workspace', + description=__doc__, + ) + + parser.add_argument( + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + parser.add_argument( + '-t', '--token', + default=default_token, + help='API token (default: TRUSTGRAPH_TOKEN environment variable)', + ) + + parser.add_argument( + '-w', '--workspace', + default=default_workspace, + help=f'Workspace to export (default: {default_workspace})', + ) + + parser.add_argument( + '-o', '--output', + required=True, + help='Output bundle path, e.g. workspace-default.tgx', + ) + + args = parser.parse_args() + + try: + + export_workspace( + url=args.api_url, + workspace=args.workspace, + output=args.output, + token=args.token, + ) + + except Exception as e: + + print("Exception:", e, flush=True) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/trustgraph-cli/trustgraph/cli/import_workspace.py b/trustgraph-cli/trustgraph/cli/import_workspace.py new file mode 100644 index 00000000..71885bfb --- /dev/null +++ b/trustgraph-cli/trustgraph/cli/import_workspace.py @@ -0,0 +1,194 @@ +""" +Imports a workspace bundle (.tgx, produced by tg-export-workspace) into a +TrustGraph deployment. The target workspace defaults to the name recorded +in the bundle's manifest and can be renamed with --workspace. + +By default existing (type, key) entries in the target workspace are left +untouched and only missing keys are added, matching WorkspaceInit's +re-run behaviour; pass --overwrite to replace every imported key. Use +--dry-run to show what would be written without changing anything. +""" + +import argparse +import json +import os +import sys +import tarfile + +from trustgraph.api import Api +from trustgraph.api.types import ConfigValue + +default_url = os.getenv("TRUSTGRAPH_URL", 'http://localhost:8088/') +default_token = os.getenv("TRUSTGRAPH_TOKEN", None) + +SUPPORTED_FORMAT = "tgx" +SUPPORTED_FORMAT_VERSION = 1 + + +def _read_bundle(path): + """Read manifest and config entries from a .tgx bundle.""" + + manifest = None + entries = [] + + with tarfile.open(path, "r:gz") as tar: + for member in tar.getmembers(): + if not member.isfile(): + continue + f = tar.extractfile(member) + if f is None: + continue + data = f.read() + if member.name == "manifest.json": + manifest = json.loads(data) + elif member.name.startswith("config/") and \ + member.name.endswith(".json"): + entries.append(json.loads(data)) + + if manifest is None: + raise RuntimeError("not a workspace bundle: manifest.json missing") + + if manifest.get("format") != SUPPORTED_FORMAT: + raise RuntimeError( + f"unsupported bundle format: {manifest.get('format')!r}" + ) + + if manifest.get("format_version", 0) > SUPPORTED_FORMAT_VERSION: + raise RuntimeError( + f"bundle format version {manifest.get('format_version')} is " + f"newer than this tool supports ({SUPPORTED_FORMAT_VERSION}); " + "upgrade trustgraph-cli" + ) + + return manifest, entries + + +def import_workspace( + url, input, workspace=None, overwrite=False, config_only=False, + dry_run=False, token=None, +): + + manifest, entries = _read_bundle(input) + + # Knowledge import (triples, documents, embeddings) is not implemented + # yet; refuse to silently drop it from a bundle that carries it. + if manifest.get("contents", {}).get("knowledge") and not config_only: + raise RuntimeError( + "bundle contains knowledge data, which this tool cannot import " + "yet; re-run with --config-only to import just the configuration" + ) + + target = workspace or manifest.get("workspace") or "default" + + api = Api(url, token=token, workspace=target).config() + + # Mirror WorkspaceInit's re-run behaviour: without --overwrite, keys + # already present in the target workspace are skipped (per key, not per + # type). The config API's put is a blanket upsert, so filter client-side. + existing = {} + if not overwrite: + for type_ in sorted({e["type"] for e in entries}): + existing[type_] = set(api.list(type_)) + + values = [] + skipped = 0 + + for e in entries: + type_, key, value = e["type"], e["key"], e["value"] + if not overwrite and key in existing.get(type_, set()): + skipped += 1 + continue + # Config values are stored as JSON strings (see WorkspaceInit). + values.append( + ConfigValue(type=type_, key=key, value=json.dumps(value)) + ) + + if dry_run: + for v in values: + print(f"would import {v.type}/{v.key}", flush=True) + print(f"Dry run: {len(values)} item(s) would be imported into " + f"workspace '{target}', {skipped} skipped as existing", + flush=True) + return + + if values: + api.put(values) + + print(f"Imported {len(values)} config item(s) into workspace " + f"'{target}', {skipped} skipped as existing", flush=True) + + +def main(): + + parser = argparse.ArgumentParser( + prog='tg-import-workspace', + description=__doc__, + ) + + parser.add_argument( + '-u', '--api-url', + default=default_url, + help=f'API URL (default: {default_url})', + ) + + parser.add_argument( + '-t', '--token', + default=default_token, + help='API token (default: TRUSTGRAPH_TOKEN environment variable)', + ) + + parser.add_argument( + '-i', '--input', + required=True, + help='Input bundle path, e.g. workspace-default.tgx', + ) + + parser.add_argument( + '-w', '--workspace', + default=None, + help='Target workspace (default: the workspace recorded in the ' + 'bundle manifest)', + ) + + parser.add_argument( + '--overwrite', + action='store_true', + help='Replace existing keys in the target workspace (default: ' + 'keep existing keys and only add missing ones)', + ) + + parser.add_argument( + '--config-only', + action='store_true', + help='Import only the configuration, skipping any knowledge data ' + 'in the bundle', + ) + + parser.add_argument( + '--dry-run', + action='store_true', + help='Show what would be imported without writing anything', + ) + + args = parser.parse_args() + + try: + + import_workspace( + url=args.api_url, + input=args.input, + workspace=args.workspace, + overwrite=args.overwrite, + config_only=args.config_only, + dry_run=args.dry_run, + token=args.token, + ) + + except Exception as e: + + print("Exception:", e, flush=True) + sys.exit(1) + + +if __name__ == "__main__": + main()