Feature/configure flows (#345)

- Keeps processing in different flows separate so that data can go to different stores / collections etc.
- Potentially supports different processing flows
- Tidies the processing API with common base-classes for e.g. LLMs, and automatic configuration of 'clients' to use the right queue names in a flow

tests/test-agent

@@ -20,7 +20,11 @@ def output(text, prefix="> ", width=78):
     )
     print(out)
 
-p = AgentClient(pulsar_host="pulsar://localhost:6650")
+p = AgentClient(
+    pulsar_host="pulsar://pulsar:6650",
+    input_queue = "non-persistent://tg/request/agent:0000",
+    output_queue = "non-persistent://tg/response/agent:0000",
+)
 
 q = "How many cats does Mark have? Calculate that number raised to 0.4 power.  Is that number lower than the numeric part of the mission identifier of the Space Shuttle Challenger on its last mission?  If so, give me an apple pie recipe, otherwise return a poem about cheese."
 

tests/test-doc-rag

@@ -3,7 +3,12 @@
 import pulsar
 from trustgraph.clients.document_rag_client import DocumentRagClient
 
-rag = DocumentRagClient(pulsar_host="pulsar://localhost:6650")
+rag = DocumentRagClient(
+    pulsar_host="pulsar://localhost:6650",
+    subscriber="test1",
+    input_queue = "non-persistent://tg/request/document-rag:default",
+    output_queue = "non-persistent://tg/response/document-rag:default",
+)
 
 query="""
 What was the cause of the space shuttle disaster?"""

tests/test-embeddings

@@ -3,7 +3,12 @@
 import pulsar
 from trustgraph.clients.embeddings_client import EmbeddingsClient
 
-embed = EmbeddingsClient(pulsar_host="pulsar://localhost:6650")
+embed = EmbeddingsClient(
+    pulsar_host="pulsar://pulsar:6650",
+    input_queue="non-persistent://tg/request/embeddings:default",
+    output_queue="non-persistent://tg/response/embeddings:default",
+    subscriber="test1",
+)
 
 prompt="Write a funny limerick about a llama"
 
@@ -11,5 +16,3 @@ resp = embed.request(prompt)
 
 print(resp)
 
-
-

tests/test-graph-rag

@@ -3,11 +3,18 @@
 import pulsar
 from trustgraph.clients.graph_rag_client import GraphRagClient
 
-rag = GraphRagClient(pulsar_host="pulsar://localhost:6650")
+rag = GraphRagClient(
+    pulsar_host="pulsar://localhost:6650",
+    subscriber="test1",
+    input_queue = "non-persistent://tg/request/graph-rag:default",
+    output_queue = "non-persistent://tg/response/graph-rag:default",
+)
 
-query="""
-This knowledge graph describes the Space Shuttle disaster.
-Present 20 facts which are present in the knowledge graph."""
+#query="""
+#This knowledge graph describes the Space Shuttle disaster.
+#Present 20 facts which are present in the knowledge graph."""
+
+query = "How many cats does Mark have?"
 
 resp = rag.request(query)
 

tests/test-llm

@@ -3,14 +3,17 @@
 import pulsar
 from trustgraph.clients.llm_client import LlmClient
 
-llm = LlmClient(pulsar_host="pulsar://localhost:6650")
+llm = LlmClient(
+    pulsar_host="pulsar://pulsar:6650",
+    input_queue="non-persistent://tg/request/text-completion:default",
+    output_queue="non-persistent://tg/response/text-completion:default",
+    subscriber="test1",
+)
 
 system = "You are a lovely assistant."
-prompt="Write a funny limerick about a llama"
+prompt="what is 2 + 2 == 5"
 
 resp = llm.request(system, prompt)
 
 print(resp)
 
-
-

tests/test-prompt-extraction

@@ -3,7 +3,12 @@
 import json
 from trustgraph.clients.prompt_client import PromptClient
 
-p = PromptClient(pulsar_host="pulsar://localhost:6650")
+p = PromptClient(
+    pulsar_host="pulsar://localhost:6650",
+    input_queue="non-persistent://tg/request/prompt:default",
+    output_queue="non-persistent://tg/response/prompt:default",
+    subscriber="test1",
+)
 
 chunk="""
  The Space Shuttle was a reusable spacecraft that transported astronauts and cargo to and from Earth's orbit. It was designed to launch like a rocket, maneuver in orbit like a spacecraft, and land like an airplane. The Space Shuttle was NASA's space transportation system and was used for many purposes, including: 
@@ -31,8 +36,8 @@ The Space Shuttle's last mission was in 2011.
 q = "Tell me some facts in the knowledge graph"
 
 resp = p.request(
-    id="extract-definition",
-    terms = {
+    id="extract-definitions",
+    variables = {
         "text": chunk,
     }
 )
@@ -40,7 +45,7 @@ resp = p.request(
 print(resp)
 
 for fact in resp:
-    print(fact["term"], "::")
+    print(fact["entity"], "::")
     print(fact["definition"])
     print()
 

tests/test-prompt-question

@@ -3,13 +3,18 @@
 import pulsar
 from trustgraph.clients.prompt_client import PromptClient
 
-p = PromptClient(pulsar_host="pulsar://localhost:6650")
+p = PromptClient(
+    pulsar_host="pulsar://localhost:6650",
+    input_queue="non-persistent://tg/request/prompt:default",
+    output_queue="non-persistent://tg/response/prompt:default",
+    subscriber="test1",
+)
 
 question = """What is the square root of 16?"""
 
 resp = p.request(
     id="question",
-    terms = {
+    variables = {
         "question": question
     }
 )

tests/test-triples

@@ -3,7 +3,9 @@
 import pulsar
 from trustgraph.clients.triples_query_client import TriplesQueryClient
 
-tq = TriplesQueryClient(pulsar_host="pulsar://localhost:6650")
+tq = TriplesQueryClient(
+    pulsar_host="pulsar://localhost:6650",
+)
 
 e = "http://trustgraph.ai/e/shuttle"