Feature/flow management cli (#346)

Flow management API + various flow management commands

trustgraph-cli/scripts/tg-delete-flow-class
trustgraph-cli/scripts/tg-get-flow-class
trustgraph-cli/scripts/tg-put-flow-class
trustgraph-cli/scripts/tg-show-flow-classes
trustgraph-cli/scripts/tg-show-flows
trustgraph-cli/scripts/tg-start-flow
trustgraph-cli/scripts/tg-stop-flow

trustgraph-flow/trustgraph/gateway/flow.py

@@ -0,0 +1,51 @@
+
+from .. schema import FlowRequest, FlowResponse, ConfigKey, ConfigValue
+from .. schema import flow_request_queue
+from .. schema import flow_response_queue
+
+from . endpoint import ServiceEndpoint
+from . requestor import ServiceRequestor
+
+class FlowRequestor(ServiceRequestor):
+    def __init__(self, pulsar_client, timeout, auth):
+
+        super(FlowRequestor, self).__init__(
+            pulsar_client=pulsar_client,
+            request_queue=flow_request_queue,
+            response_queue=flow_response_queue,
+            request_schema=FlowRequest,
+            response_schema=FlowResponse,
+            timeout=timeout,
+        )
+
+    def to_request(self, body):
+
+        return FlowRequest(
+            operation = body.get("operation", None),
+            class_name = body.get("class-name", None),
+            class_definition = body.get("class-definition", None),
+            description = body.get("description", None),
+            flow_id = body.get("flow-id", None),
+        )
+
+    def from_response(self, message):
+
+        response = { }
+
+        if message.class_names is not None:
+            response["class-names"] = message.class_names
+
+        if message.flow_ids is not None:
+            response["flow-ids"] = message.flow_ids
+
+        if message.class_definition is not None:
+            response["class-definition"] = message.class_definition
+
+        if message.flow is not None:
+            response["flow"] = message.flow
+
+        if message.description is not None:
+            response["description"] = message.description
+
+        return response, True
+

trustgraph-flow/trustgraph/gateway/mux.py

@@ -5,7 +5,6 @@ import uuid
 from aiohttp import web, WSMsgType
 
 from . socket import SocketEndpoint
-from . text_completion import TextCompletionRequestor
 
 MAX_OUTSTANDING_REQUESTS = 15
 WORKER_CLOSE_WAIT = 0.01

trustgraph-flow/trustgraph/gateway/service.py

@@ -25,28 +25,30 @@ from .. log_level import LogLevel
 
 from . serialize import to_subgraph
 from . running import Running
-from . text_completion import TextCompletionRequestor
-from . prompt import PromptRequestor
-from . graph_rag import GraphRagRequestor
-from . document_rag import DocumentRagRequestor
-from . triples_query import TriplesQueryRequestor
-from . graph_embeddings_query import GraphEmbeddingsQueryRequestor
-from . embeddings import EmbeddingsRequestor
-from . encyclopedia import EncyclopediaRequestor
-from . agent import AgentRequestor
-from . dbpedia import DbpediaRequestor
-from . internet_search import InternetSearchRequestor
-from . librarian import LibrarianRequestor
+
+#from . text_completion import TextCompletionRequestor
+#from . prompt import PromptRequestor
+#from . graph_rag import GraphRagRequestor
+#from . document_rag import DocumentRagRequestor
+#from . triples_query import TriplesQueryRequestor
+#from . graph_embeddings_query import GraphEmbeddingsQueryRequestor
+#from . embeddings import EmbeddingsRequestor
+#from . encyclopedia import EncyclopediaRequestor
+#from . agent import AgentRequestor
+#from . dbpedia import DbpediaRequestor
+#from . internet_search import InternetSearchRequestor
+#from . librarian import LibrarianRequestor
 from . config import ConfigRequestor
-from . triples_stream import TriplesStreamEndpoint
-from . graph_embeddings_stream import GraphEmbeddingsStreamEndpoint
-from . document_embeddings_stream import DocumentEmbeddingsStreamEndpoint
-from . triples_load import TriplesLoadEndpoint
-from . graph_embeddings_load import GraphEmbeddingsLoadEndpoint
-from . document_embeddings_load import DocumentEmbeddingsLoadEndpoint
+from . flow import FlowRequestor
+#from . triples_stream import TriplesStreamEndpoint
+#from . graph_embeddings_stream import GraphEmbeddingsStreamEndpoint
+#from . document_embeddings_stream import DocumentEmbeddingsStreamEndpoint
+#from . triples_load import TriplesLoadEndpoint
+#from . graph_embeddings_load import GraphEmbeddingsLoadEndpoint
+#from . document_embeddings_load import DocumentEmbeddingsLoadEndpoint
 from . mux import MuxEndpoint
-from . document_load import DocumentLoadSender
-from . text_load import TextLoadSender
+#from . document_load import DocumentLoadSender
+#from . text_load import TextLoadSender
 from . metrics import MetricsEndpoint
 
 from . endpoint import ServiceEndpoint
@@ -105,157 +107,165 @@ class Api:
             self.auth = Authenticator(allow_all=True)
 
         self.services = {
-            "text-completion": TextCompletionRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "prompt": PromptRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "graph-rag": GraphRagRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "document-rag": DocumentRagRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "triples-query": TriplesQueryRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "graph-embeddings-query": GraphEmbeddingsQueryRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "embeddings": EmbeddingsRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "agent": AgentRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "librarian": LibrarianRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
+            # "text-completion": TextCompletionRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "prompt": PromptRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "graph-rag": GraphRagRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "document-rag": DocumentRagRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "triples-query": TriplesQueryRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "graph-embeddings-query": GraphEmbeddingsQueryRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "embeddings": EmbeddingsRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "agent": AgentRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "librarian": LibrarianRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
             "config": ConfigRequestor(
                 pulsar_client=self.pulsar_client, timeout=self.timeout,
                 auth = self.auth,
             ),
-            "encyclopedia": EncyclopediaRequestor(
+            "flow": FlowRequestor(
                 pulsar_client=self.pulsar_client, timeout=self.timeout,
                 auth = self.auth,
             ),
-            "dbpedia": DbpediaRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "internet-search": InternetSearchRequestor(
-                pulsar_client=self.pulsar_client, timeout=self.timeout,
-                auth = self.auth,
-            ),
-            "document-load": DocumentLoadSender(
-                pulsar_client=self.pulsar_client,
-            ),
-            "text-load": TextLoadSender(
-                pulsar_client=self.pulsar_client,
-            ),
+            # "encyclopedia": EncyclopediaRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "dbpedia": DbpediaRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "internet-search": InternetSearchRequestor(
+            #     pulsar_client=self.pulsar_client, timeout=self.timeout,
+            #     auth = self.auth,
+            # ),
+            # "document-load": DocumentLoadSender(
+            #     pulsar_client=self.pulsar_client,
+            # ),
+            # "text-load": TextLoadSender(
+            #     pulsar_client=self.pulsar_client,
+            # ),
         }
 
         self.endpoints = [
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/text-completion", auth=self.auth,
-                requestor = self.services["text-completion"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/prompt", auth=self.auth,
-                requestor = self.services["prompt"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/graph-rag", auth=self.auth,
-                requestor = self.services["graph-rag"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/document-rag", auth=self.auth,
-                requestor = self.services["document-rag"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/triples-query", auth=self.auth,
-                requestor = self.services["triples-query"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/graph-embeddings-query",
-                auth=self.auth,
-                requestor = self.services["graph-embeddings-query"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/embeddings", auth=self.auth,
-                requestor = self.services["embeddings"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/agent", auth=self.auth,
-                requestor = self.services["agent"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/librarian", auth=self.auth,
-                requestor = self.services["librarian"],
-            ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/text-completion", auth=self.auth,
+            #     requestor = self.services["text-completion"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/prompt", auth=self.auth,
+            #     requestor = self.services["prompt"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/graph-rag", auth=self.auth,
+            #     requestor = self.services["graph-rag"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/document-rag", auth=self.auth,
+            #     requestor = self.services["document-rag"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/triples-query", auth=self.auth,
+            #     requestor = self.services["triples-query"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/graph-embeddings-query",
+            #     auth=self.auth,
+            #     requestor = self.services["graph-embeddings-query"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/embeddings", auth=self.auth,
+            #     requestor = self.services["embeddings"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/agent", auth=self.auth,
+            #     requestor = self.services["agent"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/librarian", auth=self.auth,
+            #     requestor = self.services["librarian"],
+            # ),
             ServiceEndpoint(
                 endpoint_path = "/api/v1/config", auth=self.auth,
                 requestor = self.services["config"],
             ),
             ServiceEndpoint(
-                endpoint_path = "/api/v1/encyclopedia", auth=self.auth,
-                requestor = self.services["encyclopedia"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/dbpedia", auth=self.auth,
-                requestor = self.services["dbpedia"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/internet-search", auth=self.auth,
-                requestor = self.services["internet-search"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/load/document", auth=self.auth,
-                requestor = self.services["document-load"],
-            ),
-            ServiceEndpoint(
-                endpoint_path = "/api/v1/load/text", auth=self.auth,
-                requestor = self.services["text-load"],
-            ),
-            TriplesStreamEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-            ),
-            GraphEmbeddingsStreamEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-            ),
-            DocumentEmbeddingsStreamEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-            ),
-            TriplesLoadEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-            ),
-            GraphEmbeddingsLoadEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-            ),
-            DocumentEmbeddingsLoadEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-            ),
-            MuxEndpoint(
-                pulsar_client=self.pulsar_client,
-                auth = self.auth,
-                services = self.services,
+                endpoint_path = "/api/v1/flow", auth=self.auth,
+                requestor = self.services["flow"],
             ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/encyclopedia", auth=self.auth,
+            #     requestor = self.services["encyclopedia"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/dbpedia", auth=self.auth,
+            #     requestor = self.services["dbpedia"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/internet-search", auth=self.auth,
+            #     requestor = self.services["internet-search"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/load/document", auth=self.auth,
+            #     requestor = self.services["document-load"],
+            # ),
+            # ServiceEndpoint(
+            #     endpoint_path = "/api/v1/load/text", auth=self.auth,
+            #     requestor = self.services["text-load"],
+            # ),
+            # TriplesStreamEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            # ),
+            # GraphEmbeddingsStreamEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            # ),
+            # DocumentEmbeddingsStreamEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            # ),
+            # TriplesLoadEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            # ),
+            # GraphEmbeddingsLoadEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            # ),
+            # DocumentEmbeddingsLoadEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            # ),
+            # MuxEndpoint(
+            #     pulsar_client=self.pulsar_client,
+            #     auth = self.auth,
+            #     services = self.services,
+            # ),
             MetricsEndpoint(
                 endpoint_path = "/api/v1/metrics",
                 prometheus_url = self.prometheus_url,