Trustgraph, first drop of code

scripts/chunker-recursive

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.chunker.recursive import run
+
+run()
+

scripts/embeddings-hf

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.hf import run
+
+run()
+

scripts/embeddings-vectorize

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.embeddings.vectorize import run
+
+run()
+

scripts/graph-rag

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.rag.graph import run
+
+run()
+

scripts/graph-show

@@ -0,0 +1,10 @@
+#!/usr/bin/env python3
+
+from trustgraph.trustgraph import TrustGraph
+
+t = TrustGraph()
+
+rows = t.get_all(limit=100_000_000)
+for s, p, o in rows:
+    print(s, p, o)
+

scripts/graph-to-turtle

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+
+from trustgraph.trustgraph import TrustGraph
+import rdflib
+import sys
+import io
+
+t = TrustGraph()
+
+g = rdflib.Graph()
+
+rows = t.get_all(limit=100_000_000)
+for s, p, o in rows:
+
+#     print(s, p, o)
+    sv = rdflib.term.URIRef(s)
+    pv = rdflib.term.URIRef(p)
+
+    if o.startswith("https://") or o.startswith("http://"):
+
+        # Skip malformed URLs with spaces in
+        if " " in o:
+            continue
+
+        ov = rdflib.term.URIRef(o)
+    else:
+        ov = rdflib.term.Literal(o)
+
+    g.add((sv, pv, ov))
+
+g.serialize(destination="output.ttl", format="turtle")
+
+buf = io.BytesIO()
+
+g.serialize(destination=buf, format="turtle")
+
+sys.stdout.write(buf.getvalue().decode("utf-8"))

scripts/graph-write-cassandra

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.graph.cassandra_write import run
+
+run()
+

scripts/init-pulsar-manager

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+CSRF_TOKEN=$(curl http://localhost:7750/pulsar-manager/csrf-token)
+
+curl \
+    -H "X-XSRF-TOKEN: $CSRF_TOKEN" \
+    -H "Cookie: XSRF-TOKEN=$CSRF_TOKEN;" \
+    -H 'Content-Type: application/json' \
+    -X PUT \
+    http://localhost:7750/pulsar-manager/users/superuser \
+    -d '{"name": "admin", "password": "apachepulsar", "description": "test", "email": "username@test.org"}'

scripts/kg-extract-definitions

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.kg.extract_definitions import run
+
+run()
+

scripts/kg-extract-relationships

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.kg.extract_relationships import run
+
+run()
+

scripts/llm-azure-text

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.llm.azure_text import run
+
+run()
+

scripts/llm-claude-text

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.llm.claude_text import run
+
+run()
+

scripts/llm-ollama-text

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.llm.ollama_text import run
+
+run()
+

scripts/llm-vertexai-text

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.llm.vertexai_text import run
+
+run()
+

scripts/loader

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+
+import pulsar
+from pulsar.schema import JsonSchema, Bytes, String
+from trustgraph.schema import Document, Source
+import base64
+import hashlib
+
+# client = pulsar.Client("pulsar://localhost:6650")
+host="10.89.1.246"
+host="localhost"
+client = pulsar.Client(f"pulsar://{host}:6650")
+
+producer = client.create_producer(
+    topic='document-load',
+    schema=JsonSchema(Document),
+    chunking_enabled=True,
+)
+
+files=[
+    "Challenger-Report-Vol1.pdf",
+#     "columbia-accident-investigation-board-report-volume-1.pdf",
+#     "Proposed_CIRCIA_Rules.pdf",
+]
+
+for file in files:
+
+    path = "sources/" + file
+    data = open(path, "rb").read()
+
+    id = hashlib.sha256(path.encode("utf-8")).hexdigest()[0:8]
+
+    r = Document(
+        source=Source(
+            source=path,
+            title=path,
+            id=id,
+        ),
+        data=base64.b64encode(data),
+    )
+
+    resp = producer.send(r)
+
+    print(resp)
+
+client.close()
+

scripts/pdf-decoder

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.decoder.pdf import run
+
+run()
+

scripts/query

@@ -0,0 +1,16 @@
+#!/usr/bin/env python3
+
+from trustgraph.graph_rag import GraphRag
+import sys
+
+query = " ".join(sys.argv[1:])
+
+gr = GraphRag(verbose=True)
+
+if query == "":
+    query="""This knowledge graph describes the Space Shuttle disaster.
+Present 20 facts which are present in the knowledge graph."""
+
+resp = gr.query(query)
+print(resp)
+

scripts/vector-write-milvus

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from trustgraph.vector.milvus_write import run
+
+run()
+