diff --git a/trustgraph-base/trustgraph/schema/core/metadata.py b/trustgraph-base/trustgraph/schema/core/metadata.py index cb2022ac..1888e612 100644 --- a/trustgraph-base/trustgraph/schema/core/metadata.py +++ b/trustgraph-base/trustgraph/schema/core/metadata.py @@ -1,16 +1,14 @@ - -from pulsar.schema import Record, String, Array +from dataclasses import dataclass, field from .primitives import Triple -class Metadata(Record): - +@dataclass +class Metadata: # Source identifier - id = String() + id: str = "" # Subgraph - metadata = Array(Triple()) + metadata: list[Triple] = field(default_factory=list) # Collection management - user = String() - collection = String() - + user: str = "" + collection: str = "" diff --git a/trustgraph-base/trustgraph/schema/core/primitives.py b/trustgraph-base/trustgraph/schema/core/primitives.py index fb85d05c..02517614 100644 --- a/trustgraph-base/trustgraph/schema/core/primitives.py +++ b/trustgraph-base/trustgraph/schema/core/primitives.py @@ -1,34 +1,39 @@ -from pulsar.schema import Record, String, Boolean, Array, Integer +from dataclasses import dataclass, field -class Error(Record): - type = String() - message = String() +@dataclass +class Error: + type: str = "" + message: str = "" -class Value(Record): - value = String() - is_uri = Boolean() - type = String() +@dataclass +class Value: + value: str = "" + is_uri: bool = False + type: str = "" -class Triple(Record): - s = Value() - p = Value() - o = Value() +@dataclass +class Triple: + s: Value | None = None + p: Value | None = None + o: Value | None = None -class Field(Record): - name = String() +@dataclass +class Field: + name: str = "" # int, string, long, bool, float, double, timestamp - type = String() - size = Integer() - primary = Boolean() - description = String() + type: str = "" + size: int = 0 + primary: bool = False + description: str = "" # NEW FIELDS for structured data: - required = Boolean() # Whether field is required - enum_values = Array(String()) # For enum type fields - indexed = Boolean() # Whether field should be indexed + required: bool = False # Whether field is required + enum_values: list[str] = field(default_factory=list) # For enum type fields + indexed: bool = False # Whether field should be indexed -class RowSchema(Record): - name = String() - description = String() - fields = Array(Field()) +@dataclass +class RowSchema: + name: str = "" + description: str = "" + fields: list[Field] = field(default_factory=list) diff --git a/trustgraph-base/trustgraph/schema/core/topic.py b/trustgraph-base/trustgraph/schema/core/topic.py index cdd643b7..09c633e4 100644 --- a/trustgraph-base/trustgraph/schema/core/topic.py +++ b/trustgraph-base/trustgraph/schema/core/topic.py @@ -1,4 +1,23 @@ -def topic(topic, kind='persistent', tenant='tg', namespace='flow'): - return f"{kind}://{tenant}/{namespace}/{topic}" +def topic(queue_name, qos='q1', tenant='tg', namespace='flow'): + """ + Create a generic topic identifier that can be mapped by backends. + + Args: + queue_name: The queue/topic name + qos: Quality of service + - 'q0' = best-effort (no ack) + - 'q1' = at-least-once (ack required) + - 'q2' = exactly-once (two-phase ack) + tenant: Tenant identifier for multi-tenancy + namespace: Namespace within tenant + + Returns: + Generic topic string: qos/tenant/namespace/queue_name + + Examples: + topic('my-queue') # q1/tg/flow/my-queue + topic('config', qos='q2', namespace='config') # q2/tg/config/config + """ + return f"{qos}/{tenant}/{namespace}/{queue_name}" diff --git a/trustgraph-base/trustgraph/schema/knowledge/document.py b/trustgraph-base/trustgraph/schema/knowledge/document.py index f41ee8a6..d8ce97b4 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/document.py +++ b/trustgraph-base/trustgraph/schema/knowledge/document.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, Bytes +from dataclasses import dataclass from ..core.metadata import Metadata from ..core.topic import topic @@ -6,24 +6,27 @@ from ..core.topic import topic ############################################################################ # PDF docs etc. -class Document(Record): - metadata = Metadata() - data = Bytes() +@dataclass +class Document: + metadata: Metadata | None = None + data: bytes = b"" ############################################################################ # Text documents / text from PDF -class TextDocument(Record): - metadata = Metadata() - text = Bytes() +@dataclass +class TextDocument: + metadata: Metadata | None = None + text: bytes = b"" ############################################################################ # Chunks of text -class Chunk(Record): - metadata = Metadata() - chunk = Bytes() +@dataclass +class Chunk: + metadata: Metadata | None = None + chunk: bytes = b"" -############################################################################ \ No newline at end of file +############################################################################ diff --git a/trustgraph-base/trustgraph/schema/knowledge/embeddings.py b/trustgraph-base/trustgraph/schema/knowledge/embeddings.py index cfdae068..a3e5b394 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/embeddings.py +++ b/trustgraph-base/trustgraph/schema/knowledge/embeddings.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double, Map +from dataclasses import dataclass, field from ..core.metadata import Metadata from ..core.primitives import Value, RowSchema @@ -8,49 +8,55 @@ from ..core.topic import topic # Graph embeddings are embeddings associated with a graph entity -class EntityEmbeddings(Record): - entity = Value() - vectors = Array(Array(Double())) +@dataclass +class EntityEmbeddings: + entity: Value | None = None + vectors: list[list[float]] = field(default_factory=list) # This is a 'batching' mechanism for the above data -class GraphEmbeddings(Record): - metadata = Metadata() - entities = Array(EntityEmbeddings()) +@dataclass +class GraphEmbeddings: + metadata: Metadata | None = None + entities: list[EntityEmbeddings] = field(default_factory=list) ############################################################################ # Document embeddings are embeddings associated with a chunk -class ChunkEmbeddings(Record): - chunk = Bytes() - vectors = Array(Array(Double())) +@dataclass +class ChunkEmbeddings: + chunk: bytes = b"" + vectors: list[list[float]] = field(default_factory=list) # This is a 'batching' mechanism for the above data -class DocumentEmbeddings(Record): - metadata = Metadata() - chunks = Array(ChunkEmbeddings()) +@dataclass +class DocumentEmbeddings: + metadata: Metadata | None = None + chunks: list[ChunkEmbeddings] = field(default_factory=list) ############################################################################ # Object embeddings are embeddings associated with the primary key of an # object -class ObjectEmbeddings(Record): - metadata = Metadata() - vectors = Array(Array(Double())) - name = String() - key_name = String() - id = String() +@dataclass +class ObjectEmbeddings: + metadata: Metadata | None = None + vectors: list[list[float]] = field(default_factory=list) + name: str = "" + key_name: str = "" + id: str = "" ############################################################################ # Structured object embeddings with enhanced capabilities -class StructuredObjectEmbedding(Record): - metadata = Metadata() - vectors = Array(Array(Double())) - schema_name = String() - object_id = String() # Primary key value - field_embeddings = Map(Array(Double())) # Per-field embeddings +@dataclass +class StructuredObjectEmbedding: + metadata: Metadata | None = None + vectors: list[list[float]] = field(default_factory=list) + schema_name: str = "" + object_id: str = "" # Primary key value + field_embeddings: dict[str, list[float]] = field(default_factory=dict) # Per-field embeddings -############################################################################ \ No newline at end of file +############################################################################ diff --git a/trustgraph-base/trustgraph/schema/knowledge/graph.py b/trustgraph-base/trustgraph/schema/knowledge/graph.py index 1d55c8f0..9040c25e 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/graph.py +++ b/trustgraph-base/trustgraph/schema/knowledge/graph.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Array +from dataclasses import dataclass, field from ..core.primitives import Value, Triple from ..core.metadata import Metadata @@ -8,21 +8,24 @@ from ..core.topic import topic # Entity context are an entity associated with textual context -class EntityContext(Record): - entity = Value() - context = String() +@dataclass +class EntityContext: + entity: Value | None = None + context: str = "" # This is a 'batching' mechanism for the above data -class EntityContexts(Record): - metadata = Metadata() - entities = Array(EntityContext()) +@dataclass +class EntityContexts: + metadata: Metadata | None = None + entities: list[EntityContext] = field(default_factory=list) ############################################################################ # Graph triples -class Triples(Record): - metadata = Metadata() - triples = Array(Triple()) +@dataclass +class Triples: + metadata: Metadata | None = None + triples: list[Triple] = field(default_factory=list) -############################################################################ \ No newline at end of file +############################################################################ diff --git a/trustgraph-base/trustgraph/schema/knowledge/knowledge.py b/trustgraph-base/trustgraph/schema/knowledge/knowledge.py index 7cd5450e..cffcbac7 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/knowledge.py +++ b/trustgraph-base/trustgraph/schema/knowledge/knowledge.py @@ -1,5 +1,4 @@ - -from pulsar.schema import Record, Bytes, String, Array, Long, Boolean +from dataclasses import dataclass, field from ..core.primitives import Triple, Error from ..core.topic import topic from ..core.metadata import Metadata @@ -22,40 +21,40 @@ from .embeddings import GraphEmbeddings # <- () # <- (error) -class KnowledgeRequest(Record): - +@dataclass +class KnowledgeRequest: # get-kg-core, delete-kg-core, list-kg-cores, put-kg-core # load-kg-core, unload-kg-core - operation = String() + operation: str = "" # list-kg-cores, delete-kg-core, put-kg-core - user = String() + user: str = "" # get-kg-core, list-kg-cores, delete-kg-core, put-kg-core, # load-kg-core, unload-kg-core - id = String() + id: str = "" # load-kg-core - flow = String() + flow: str = "" # load-kg-core - collection = String() + collection: str = "" # put-kg-core - triples = Triples() - graph_embeddings = GraphEmbeddings() + triples: Triples | None = None + graph_embeddings: GraphEmbeddings | None = None -class KnowledgeResponse(Record): - error = Error() - ids = Array(String()) - eos = Boolean() # Indicates end of knowledge core stream - triples = Triples() - graph_embeddings = GraphEmbeddings() +@dataclass +class KnowledgeResponse: + error: Error | None = None + ids: list[str] = field(default_factory=list) + eos: bool = False # Indicates end of knowledge core stream + triples: Triples | None = None + graph_embeddings: GraphEmbeddings | None = None knowledge_request_queue = topic( - 'knowledge', kind='non-persistent', namespace='request' + 'knowledge', qos='q0', namespace='request' ) knowledge_response_queue = topic( - 'knowledge', kind='non-persistent', namespace='response', + 'knowledge', qos='q0', namespace='response', ) - diff --git a/trustgraph-base/trustgraph/schema/knowledge/nlp.py b/trustgraph-base/trustgraph/schema/knowledge/nlp.py index 0ffc3ba1..10b5f215 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/nlp.py +++ b/trustgraph-base/trustgraph/schema/knowledge/nlp.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Boolean +from dataclasses import dataclass from ..core.topic import topic @@ -6,21 +6,25 @@ from ..core.topic import topic # NLP extraction data types -class Definition(Record): - name = String() - definition = String() +@dataclass +class Definition: + name: str = "" + definition: str = "" -class Topic(Record): - name = String() - definition = String() +@dataclass +class Topic: + name: str = "" + definition: str = "" -class Relationship(Record): - s = String() - p = String() - o = String() - o_entity = Boolean() +@dataclass +class Relationship: + s: str = "" + p: str = "" + o: str = "" + o_entity: bool = False -class Fact(Record): - s = String() - p = String() - o = String() \ No newline at end of file +@dataclass +class Fact: + s: str = "" + p: str = "" + o: str = "" diff --git a/trustgraph-base/trustgraph/schema/knowledge/object.py b/trustgraph-base/trustgraph/schema/knowledge/object.py index 537eb95e..39b0095f 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/object.py +++ b/trustgraph-base/trustgraph/schema/knowledge/object.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Map, Double, Array +from dataclasses import dataclass, field from ..core.metadata import Metadata from ..core.topic import topic @@ -7,11 +7,13 @@ from ..core.topic import topic # Extracted object from text processing -class ExtractedObject(Record): - metadata = Metadata() - schema_name = String() # Which schema this object belongs to - values = Array(Map(String())) # Array of objects, each object is field name -> value - confidence = Double() - source_span = String() # Text span where object was found +@dataclass +class ExtractedObject: + metadata: Metadata | None = None + schema_name: str = "" # Which schema this object belongs to + values: list[dict[str, str]] = field(default_factory=list) # Array of objects, each object is field name -> value + confidence: float = 0.0 + source_span: str = "" # Text span where object was found + +############################################################################ -############################################################################ \ No newline at end of file diff --git a/trustgraph-base/trustgraph/schema/knowledge/rows.py b/trustgraph-base/trustgraph/schema/knowledge/rows.py index 8b1c79ef..ca2131df 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/rows.py +++ b/trustgraph-base/trustgraph/schema/knowledge/rows.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, Array, Map, String +from dataclasses import dataclass, field from ..core.metadata import Metadata from ..core.primitives import RowSchema @@ -8,9 +8,10 @@ from ..core.topic import topic # Stores rows of information -class Rows(Record): - metadata = Metadata() - row_schema = RowSchema() - rows = Array(Map(String())) +@dataclass +class Rows: + metadata: Metadata | None = None + row_schema: RowSchema | None = None + rows: list[dict[str, str]] = field(default_factory=list) -############################################################################ \ No newline at end of file +############################################################################ diff --git a/trustgraph-base/trustgraph/schema/knowledge/structured.py b/trustgraph-base/trustgraph/schema/knowledge/structured.py index 3d2b1311..c227d767 100644 --- a/trustgraph-base/trustgraph/schema/knowledge/structured.py +++ b/trustgraph-base/trustgraph/schema/knowledge/structured.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Bytes, Map +from dataclasses import dataclass, field from ..core.metadata import Metadata from ..core.topic import topic @@ -7,11 +7,13 @@ from ..core.topic import topic # Structured data submission for fire-and-forget processing -class StructuredDataSubmission(Record): - metadata = Metadata() - format = String() # "json", "csv", "xml" - schema_name = String() # Reference to schema in config - data = Bytes() # Raw data to ingest - options = Map(String()) # Format-specific options +@dataclass +class StructuredDataSubmission: + metadata: Metadata | None = None + format: str = "" # "json", "csv", "xml" + schema_name: str = "" # Reference to schema in config + data: bytes = b"" # Raw data to ingest + options: dict[str, str] = field(default_factory=dict) # Format-specific options + +############################################################################ -############################################################################ \ No newline at end of file diff --git a/trustgraph-base/trustgraph/schema/services/agent.py b/trustgraph-base/trustgraph/schema/services/agent.py index 6e8be5eb..a6168eed 100644 --- a/trustgraph-base/trustgraph/schema/services/agent.py +++ b/trustgraph-base/trustgraph/schema/services/agent.py @@ -1,5 +1,5 @@ -from pulsar.schema import Record, String, Array, Map, Boolean +from dataclasses import dataclass, field from ..core.topic import topic from ..core.primitives import Error @@ -8,33 +8,36 @@ from ..core.primitives import Error # Prompt services, abstract the prompt generation -class AgentStep(Record): - thought = String() - action = String() - arguments = Map(String()) - observation = String() - user = String() # User context for the step +@dataclass +class AgentStep: + thought: str = "" + action: str = "" + arguments: dict[str, str] = field(default_factory=dict) + observation: str = "" + user: str = "" # User context for the step -class AgentRequest(Record): - question = String() - state = String() - group = Array(String()) - history = Array(AgentStep()) - user = String() # User context for multi-tenancy - streaming = Boolean() # NEW: Enable streaming response delivery (default false) +@dataclass +class AgentRequest: + question: str = "" + state: str = "" + group: list[str] = field(default_factory=list) + history: list['AgentStep'] = field(default_factory=list) + user: str = "" # User context for multi-tenancy + streaming: bool = False # NEW: Enable streaming response delivery (default false) -class AgentResponse(Record): +@dataclass +class AgentResponse: # Streaming-first design - chunk_type = String() # "thought", "action", "observation", "answer", "error" - content = String() # The actual content (interpretation depends on chunk_type) - end_of_message = Boolean() # Current chunk type (thought/action/etc.) is complete - end_of_dialog = Boolean() # Entire agent dialog is complete + chunk_type: str = "" # "thought", "action", "observation", "answer", "error" + content: str = "" # The actual content (interpretation depends on chunk_type) + end_of_message: bool = False # Current chunk type (thought/action/etc.) is complete + end_of_dialog: bool = False # Entire agent dialog is complete # Legacy fields (deprecated but kept for backward compatibility) - answer = String() - error = Error() - thought = String() - observation = String() + answer: str = "" + error: Error | None = None + thought: str = "" + observation: str = "" ############################################################################ diff --git a/trustgraph-base/trustgraph/schema/services/collection.py b/trustgraph-base/trustgraph/schema/services/collection.py index 04f644e8..74381abb 100644 --- a/trustgraph-base/trustgraph/schema/services/collection.py +++ b/trustgraph-base/trustgraph/schema/services/collection.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Integer, Array +from dataclasses import dataclass, field from datetime import datetime from ..core.primitives import Error @@ -10,37 +10,40 @@ from ..core.topic import topic # Collection metadata operations (for librarian service) -class CollectionMetadata(Record): +@dataclass +class CollectionMetadata: """Collection metadata record""" - user = String() - collection = String() - name = String() - description = String() - tags = Array(String()) + user: str = "" + collection: str = "" + name: str = "" + description: str = "" + tags: list[str] = field(default_factory=list) ############################################################################ -class CollectionManagementRequest(Record): +@dataclass +class CollectionManagementRequest: """Request for collection management operations""" - operation = String() # e.g., "delete-collection" + operation: str = "" # e.g., "delete-collection" # For 'list-collections' - user = String() - collection = String() - timestamp = String() # ISO timestamp - name = String() - description = String() - tags = Array(String()) + user: str = "" + collection: str = "" + timestamp: str = "" # ISO timestamp + name: str = "" + description: str = "" + tags: list[str] = field(default_factory=list) # For list - tag_filter = Array(String()) # Optional filter by tags - limit = Integer() + tag_filter: list[str] = field(default_factory=list) # Optional filter by tags + limit: int = 0 -class CollectionManagementResponse(Record): +@dataclass +class CollectionManagementResponse: """Response for collection management operations""" - error = Error() # Only populated if there's an error - timestamp = String() # ISO timestamp - collections = Array(CollectionMetadata()) + error: Error | None = None # Only populated if there's an error + timestamp: str = "" # ISO timestamp + collections: list[CollectionMetadata] = field(default_factory=list) ############################################################################ @@ -48,8 +51,9 @@ class CollectionManagementResponse(Record): # Topics collection_request_queue = topic( - 'collection', kind='non-persistent', namespace='request' + 'collection', qos='q0', namespace='request' ) collection_response_queue = topic( - 'collection', kind='non-persistent', namespace='response' + 'collection', qos='q0', namespace='response' ) + diff --git a/trustgraph-base/trustgraph/schema/services/config.py b/trustgraph-base/trustgraph/schema/services/config.py index a0955eab..38bd1cbf 100644 --- a/trustgraph-base/trustgraph/schema/services/config.py +++ b/trustgraph-base/trustgraph/schema/services/config.py @@ -1,5 +1,5 @@ -from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer +from dataclasses import dataclass, field from ..core.topic import topic from ..core.primitives import Error @@ -13,58 +13,61 @@ from ..core.primitives import Error # put(values) -> () # delete(keys) -> () # config() -> (version, config) -class ConfigKey(Record): - type = String() - key = String() +@dataclass +class ConfigKey: + type: str = "" + key: str = "" -class ConfigValue(Record): - type = String() - key = String() - value = String() +@dataclass +class ConfigValue: + type: str = "" + key: str = "" + value: str = "" # Prompt services, abstract the prompt generation -class ConfigRequest(Record): - - operation = String() # get, list, getvalues, delete, put, config +@dataclass +class ConfigRequest: + operation: str = "" # get, list, getvalues, delete, put, config # get, delete - keys = Array(ConfigKey()) + keys: list[ConfigKey] = field(default_factory=list) # list, getvalues - type = String() + type: str = "" # put - values = Array(ConfigValue()) - -class ConfigResponse(Record): + values: list[ConfigValue] = field(default_factory=list) +@dataclass +class ConfigResponse: # get, list, getvalues, config - version = Integer() + version: int = 0 # get, getvalues - values = Array(ConfigValue()) + values: list[ConfigValue] = field(default_factory=list) # list - directory = Array(String()) + directory: list[str] = field(default_factory=list) # config - config = Map(Map(String())) + config: dict[str, dict[str, str]] = field(default_factory=dict) # Everything - error = Error() + error: Error | None = None -class ConfigPush(Record): - version = Integer() - config = Map(Map(String())) +@dataclass +class ConfigPush: + version: int = 0 + config: dict[str, dict[str, str]] = field(default_factory=dict) config_request_queue = topic( - 'config', kind='non-persistent', namespace='request' + 'config', qos='q0', namespace='request' ) config_response_queue = topic( - 'config', kind='non-persistent', namespace='response' + 'config', qos='q0', namespace='response' ) config_push_queue = topic( - 'config', kind='persistent', namespace='config' + 'config', qos='q2', namespace='config' ) ############################################################################ diff --git a/trustgraph-base/trustgraph/schema/services/diagnosis.py b/trustgraph-base/trustgraph/schema/services/diagnosis.py index 1bd6d3ed..529e7d9e 100644 --- a/trustgraph-base/trustgraph/schema/services/diagnosis.py +++ b/trustgraph-base/trustgraph/schema/services/diagnosis.py @@ -1,33 +1,36 @@ -from pulsar.schema import Record, String, Map, Double, Array +from dataclasses import dataclass, field from ..core.primitives import Error ############################################################################ # Structured data diagnosis services -class StructuredDataDiagnosisRequest(Record): - operation = String() # "detect-type", "generate-descriptor", "diagnose", or "schema-selection" - sample = String() # Data sample to analyze (text content) - type = String() # Data type (csv, json, xml) - optional, required for generate-descriptor - schema_name = String() # Target schema name for descriptor generation - optional +@dataclass +class StructuredDataDiagnosisRequest: + operation: str = "" # "detect-type", "generate-descriptor", "diagnose", or "schema-selection" + sample: str = "" # Data sample to analyze (text content) + type: str = "" # Data type (csv, json, xml) - optional, required for generate-descriptor + schema_name: str = "" # Target schema name for descriptor generation - optional # JSON encoded options (e.g., delimiter for CSV) - options = Map(String()) + options: dict[str, str] = field(default_factory=dict) -class StructuredDataDiagnosisResponse(Record): - error = Error() +@dataclass +class StructuredDataDiagnosisResponse: + error: Error | None = None - operation = String() # The operation that was performed - detected_type = String() # Detected data type (for detect-type/diagnose) - optional - confidence = Double() # Confidence score for type detection - optional + operation: str = "" # The operation that was performed + detected_type: str = "" # Detected data type (for detect-type/diagnose) - optional + confidence: float = 0.0 # Confidence score for type detection - optional # JSON encoded descriptor (for generate-descriptor/diagnose) - optional - descriptor = String() + descriptor: str = "" # JSON encoded additional metadata (e.g., field count, sample records) - metadata = Map(String()) + metadata: dict[str, str] = field(default_factory=dict) # Array of matching schema IDs (for schema-selection operation) - optional - schema_matches = Array(String()) + schema_matches: list[str] = field(default_factory=list) + +############################################################################ -############################################################################ \ No newline at end of file diff --git a/trustgraph-base/trustgraph/schema/services/flow.py b/trustgraph-base/trustgraph/schema/services/flow.py index d03e559b..b993b1b3 100644 --- a/trustgraph-base/trustgraph/schema/services/flow.py +++ b/trustgraph-base/trustgraph/schema/services/flow.py @@ -1,5 +1,5 @@ -from pulsar.schema import Record, Bytes, String, Boolean, Array, Map, Integer +from dataclasses import dataclass, field from ..core.topic import topic from ..core.primitives import Error @@ -11,61 +11,61 @@ from ..core.primitives import Error # get_class(classname) -> (class) # put_class(class) -> (class) # delete_class(classname) -> () -# +# # list_flows() -> (flowid[]) # get_flow(flowid) -> (flow) # start_flow(flowid, classname) -> () # stop_flow(flowid) -> () # Prompt services, abstract the prompt generation -class FlowRequest(Record): - - operation = String() # list-classes, get-class, put-class, delete-class +@dataclass +class FlowRequest: + operation: str = "" # list-classes, get-class, put-class, delete-class # list-flows, get-flow, start-flow, stop-flow # get_class, put_class, delete_class, start_flow - class_name = String() + class_name: str = "" # put_class - class_definition = String() + class_definition: str = "" # start_flow - description = String() + description: str = "" # get_flow, start_flow, stop_flow - flow_id = String() + flow_id: str = "" # start_flow - optional parameters for flow customization - parameters = Map(String()) - -class FlowResponse(Record): + parameters: dict[str, str] = field(default_factory=dict) +@dataclass +class FlowResponse: # list_classes - class_names = Array(String()) + class_names: list[str] = field(default_factory=list) # list_flows - flow_ids = Array(String()) + flow_ids: list[str] = field(default_factory=list) # get_class - class_definition = String() + class_definition: str = "" # get_flow - flow = String() + flow: str = "" # get_flow - description = String() + description: str = "" # get_flow - parameters used when flow was started - parameters = Map(String()) + parameters: dict[str, str] = field(default_factory=dict) # Everything - error = Error() + error: Error | None = None flow_request_queue = topic( - 'flow', kind='non-persistent', namespace='request' + 'flow', qos='q0', namespace='request' ) flow_response_queue = topic( - 'flow', kind='non-persistent', namespace='response' + 'flow', qos='q0', namespace='response' ) ############################################################################ diff --git a/trustgraph-base/trustgraph/schema/services/library.py b/trustgraph-base/trustgraph/schema/services/library.py index d9678a90..391d49e1 100644 --- a/trustgraph-base/trustgraph/schema/services/library.py +++ b/trustgraph-base/trustgraph/schema/services/library.py @@ -1,9 +1,8 @@ - -from pulsar.schema import Record, Bytes, String, Array, Long +from dataclasses import dataclass, field from ..core.primitives import Triple, Error from ..core.topic import topic from ..core.metadata import Metadata -from ..knowledge.document import Document, TextDocument +# Note: Document imports will be updated after knowledge schemas are converted # add-document # -> (document_id, document_metadata, content) @@ -50,76 +49,79 @@ from ..knowledge.document import Document, TextDocument # <- (processing_metadata[]) # <- (error) -class DocumentMetadata(Record): - id = String() - time = Long() - kind = String() - title = String() - comments = String() - metadata = Array(Triple()) - user = String() - tags = Array(String()) +@dataclass +class DocumentMetadata: + id: str = "" + time: int = 0 + kind: str = "" + title: str = "" + comments: str = "" + metadata: list[Triple] = field(default_factory=list) + user: str = "" + tags: list[str] = field(default_factory=list) -class ProcessingMetadata(Record): - id = String() - document_id = String() - time = Long() - flow = String() - user = String() - collection = String() - tags = Array(String()) +@dataclass +class ProcessingMetadata: + id: str = "" + document_id: str = "" + time: int = 0 + flow: str = "" + user: str = "" + collection: str = "" + tags: list[str] = field(default_factory=list) -class Criteria(Record): - key = String() - value = String() - operator = String() - -class LibrarianRequest(Record): +@dataclass +class Criteria: + key: str = "" + value: str = "" + operator: str = "" +@dataclass +class LibrarianRequest: # add-document, remove-document, update-document, get-document-metadata, # get-document-content, add-processing, remove-processing, list-documents, # list-processing - operation = String() + operation: str = "" # add-document, remove-document, update-document, get-document-metadata, # get-document-content - document_id = String() + document_id: str = "" # add-processing, remove-processing - processing_id = String() + processing_id: str = "" # add-document, update-document - document_metadata = DocumentMetadata() + document_metadata: DocumentMetadata | None = None # add-processing - processing_metadata = ProcessingMetadata() + processing_metadata: ProcessingMetadata | None = None # add-document - content = Bytes() + content: bytes = b"" # list-documents, list-processing - user = String() + user: str = "" # list-documents?, list-processing? - collection = String() + collection: str = "" - # - criteria = Array(Criteria()) + # + criteria: list[Criteria] = field(default_factory=list) -class LibrarianResponse(Record): - error = Error() - document_metadata = DocumentMetadata() - content = Bytes() - document_metadatas = Array(DocumentMetadata()) - processing_metadatas = Array(ProcessingMetadata()) +@dataclass +class LibrarianResponse: + error: Error | None = None + document_metadata: DocumentMetadata | None = None + content: bytes = b"" + document_metadatas: list[DocumentMetadata] = field(default_factory=list) + processing_metadatas: list[ProcessingMetadata] = field(default_factory=list) # FIXME: Is this right? Using persistence on librarian so that # message chunking works librarian_request_queue = topic( - 'librarian', kind='persistent', namespace='request' + 'librarian', qos='q1', namespace='request' ) librarian_response_queue = topic( - 'librarian', kind='persistent', namespace='response', + 'librarian', qos='q1', namespace='response', ) - diff --git a/trustgraph-base/trustgraph/schema/services/llm.py b/trustgraph-base/trustgraph/schema/services/llm.py index 3fd21937..1261158e 100644 --- a/trustgraph-base/trustgraph/schema/services/llm.py +++ b/trustgraph-base/trustgraph/schema/services/llm.py @@ -1,5 +1,5 @@ -from pulsar.schema import Record, String, Array, Double, Integer, Boolean +from dataclasses import dataclass, field from ..core.topic import topic from ..core.primitives import Error @@ -8,46 +8,49 @@ from ..core.primitives import Error # LLM text completion -class TextCompletionRequest(Record): - system = String() - prompt = String() - streaming = Boolean() # Default false for backward compatibility +@dataclass +class TextCompletionRequest: + system: str = "" + prompt: str = "" + streaming: bool = False # Default false for backward compatibility -class TextCompletionResponse(Record): - error = Error() - response = String() - in_token = Integer() - out_token = Integer() - model = String() - end_of_stream = Boolean() # Indicates final message in stream +@dataclass +class TextCompletionResponse: + error: Error | None = None + response: str = "" + in_token: int = 0 + out_token: int = 0 + model: str = "" + end_of_stream: bool = False # Indicates final message in stream ############################################################################ # Embeddings -class EmbeddingsRequest(Record): - text = String() +@dataclass +class EmbeddingsRequest: + text: str = "" -class EmbeddingsResponse(Record): - error = Error() - vectors = Array(Array(Double())) +@dataclass +class EmbeddingsResponse: + error: Error | None = None + vectors: list[list[float]] = field(default_factory=list) ############################################################################ # Tool request/response -class ToolRequest(Record): - name = String() - +@dataclass +class ToolRequest: + name: str = "" # Parameters are JSON encoded - parameters = String() - -class ToolResponse(Record): - error = Error() + parameters: str = "" +@dataclass +class ToolResponse: + error: Error | None = None # Plain text aka "unstructured" - text = String() - + text: str = "" # JSON-encoded object aka "structured" - object = String() + object: str = "" diff --git a/trustgraph-base/trustgraph/schema/services/lookup.py b/trustgraph-base/trustgraph/schema/services/lookup.py index 7cc0bd03..bdeac636 100644 --- a/trustgraph-base/trustgraph/schema/services/lookup.py +++ b/trustgraph-base/trustgraph/schema/services/lookup.py @@ -1,5 +1,4 @@ - -from pulsar.schema import Record, String +from dataclasses import dataclass from ..core.primitives import Error, Value, Triple from ..core.topic import topic @@ -9,13 +8,14 @@ from ..core.metadata import Metadata # Lookups -class LookupRequest(Record): - kind = String() - term = String() +@dataclass +class LookupRequest: + kind: str = "" + term: str = "" -class LookupResponse(Record): - text = String() - error = Error() +@dataclass +class LookupResponse: + text: str = "" + error: Error | None = None ############################################################################ - diff --git a/trustgraph-base/trustgraph/schema/services/nlp_query.py b/trustgraph-base/trustgraph/schema/services/nlp_query.py index a3e709a1..6cd65f0e 100644 --- a/trustgraph-base/trustgraph/schema/services/nlp_query.py +++ b/trustgraph-base/trustgraph/schema/services/nlp_query.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Array, Map, Integer, Double +from dataclasses import dataclass, field from ..core.primitives import Error from ..core.topic import topic @@ -7,15 +7,18 @@ from ..core.topic import topic # NLP to Structured Query Service - converts natural language to GraphQL -class QuestionToStructuredQueryRequest(Record): - question = String() - max_results = Integer() +@dataclass +class QuestionToStructuredQueryRequest: + question: str = "" + max_results: int = 0 -class QuestionToStructuredQueryResponse(Record): - error = Error() - graphql_query = String() # Generated GraphQL query - variables = Map(String()) # GraphQL variables if any - detected_schemas = Array(String()) # Which schemas the query targets - confidence = Double() +@dataclass +class QuestionToStructuredQueryResponse: + error: Error | None = None + graphql_query: str = "" # Generated GraphQL query + variables: dict[str, str] = field(default_factory=dict) # GraphQL variables if any + detected_schemas: list[str] = field(default_factory=list) # Which schemas the query targets + confidence: float = 0.0 ############################################################################ + diff --git a/trustgraph-base/trustgraph/schema/services/objects_query.py b/trustgraph-base/trustgraph/schema/services/objects_query.py index 6c3a307c..bbb1cb61 100644 --- a/trustgraph-base/trustgraph/schema/services/objects_query.py +++ b/trustgraph-base/trustgraph/schema/services/objects_query.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Map, Array +from dataclasses import dataclass, field from ..core.primitives import Error from ..core.topic import topic @@ -7,22 +7,25 @@ from ..core.topic import topic # Objects Query Service - executes GraphQL queries against structured data -class GraphQLError(Record): - message = String() - path = Array(String()) # Path to the field that caused the error - extensions = Map(String()) # Additional error metadata +@dataclass +class GraphQLError: + message: str = "" + path: list[str] = field(default_factory=list) # Path to the field that caused the error + extensions: dict[str, str] = field(default_factory=dict) # Additional error metadata -class ObjectsQueryRequest(Record): - user = String() # Cassandra keyspace (follows pattern from TriplesQueryRequest) - collection = String() # Data collection identifier (required for partition key) - query = String() # GraphQL query string - variables = Map(String()) # GraphQL variables - operation_name = String() # Operation to execute for multi-operation documents +@dataclass +class ObjectsQueryRequest: + user: str = "" # Cassandra keyspace (follows pattern from TriplesQueryRequest) + collection: str = "" # Data collection identifier (required for partition key) + query: str = "" # GraphQL query string + variables: dict[str, str] = field(default_factory=dict) # GraphQL variables + operation_name: str = "" # Operation to execute for multi-operation documents -class ObjectsQueryResponse(Record): - error = Error() # System-level error (connection, timeout, etc.) - data = String() # JSON-encoded GraphQL response data - errors = Array(GraphQLError()) # GraphQL field-level errors - extensions = Map(String()) # Query metadata (execution time, etc.) +@dataclass +class ObjectsQueryResponse: + error: Error | None = None # System-level error (connection, timeout, etc.) + data: str = "" # JSON-encoded GraphQL response data + errors: list[GraphQLError] = field(default_factory=list) # GraphQL field-level errors + extensions: dict[str, str] = field(default_factory=dict) # Query metadata (execution time, etc.) -############################################################################ \ No newline at end of file +############################################################################ diff --git a/trustgraph-base/trustgraph/schema/services/prompt.py b/trustgraph-base/trustgraph/schema/services/prompt.py index edb569c9..f7a31c14 100644 --- a/trustgraph-base/trustgraph/schema/services/prompt.py +++ b/trustgraph-base/trustgraph/schema/services/prompt.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Map, Boolean +from dataclasses import dataclass, field from ..core.primitives import Error from ..core.topic import topic @@ -18,27 +18,28 @@ from ..core.topic import topic # extract-rows # schema, chunk -> rows -class PromptRequest(Record): - id = String() +@dataclass +class PromptRequest: + id: str = "" # JSON encoded values - terms = Map(String()) + terms: dict[str, str] = field(default_factory=dict) # Streaming support (default false for backward compatibility) - streaming = Boolean() - -class PromptResponse(Record): + streaming: bool = False +@dataclass +class PromptResponse: # Error case - error = Error() + error: Error | None = None # Just plain text - text = String() + text: str = "" # JSON encoded - object = String() + object: str = "" # Indicates final message in stream - end_of_stream = Boolean() + end_of_stream: bool = False ############################################################################ \ No newline at end of file diff --git a/trustgraph-base/trustgraph/schema/services/query.py b/trustgraph-base/trustgraph/schema/services/query.py index 91231ade..31d0852d 100644 --- a/trustgraph-base/trustgraph/schema/services/query.py +++ b/trustgraph-base/trustgraph/schema/services/query.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Integer, Array, Double +from dataclasses import dataclass, field from ..core.primitives import Error, Value, Triple from ..core.topic import topic @@ -7,49 +7,55 @@ from ..core.topic import topic # Graph embeddings query -class GraphEmbeddingsRequest(Record): - vectors = Array(Array(Double())) - limit = Integer() - user = String() - collection = String() +@dataclass +class GraphEmbeddingsRequest: + vectors: list[list[float]] = field(default_factory=list) + limit: int = 0 + user: str = "" + collection: str = "" -class GraphEmbeddingsResponse(Record): - error = Error() - entities = Array(Value()) +@dataclass +class GraphEmbeddingsResponse: + error: Error | None = None + entities: list[Value] = field(default_factory=list) ############################################################################ # Graph triples query -class TriplesQueryRequest(Record): - user = String() - collection = String() - s = Value() - p = Value() - o = Value() - limit = Integer() +@dataclass +class TriplesQueryRequest: + user: str = "" + collection: str = "" + s: Value | None = None + p: Value | None = None + o: Value | None = None + limit: int = 0 -class TriplesQueryResponse(Record): - error = Error() - triples = Array(Triple()) +@dataclass +class TriplesQueryResponse: + error: Error | None = None + triples: list[Triple] = field(default_factory=list) ############################################################################ # Doc embeddings query -class DocumentEmbeddingsRequest(Record): - vectors = Array(Array(Double())) - limit = Integer() - user = String() - collection = String() +@dataclass +class DocumentEmbeddingsRequest: + vectors: list[list[float]] = field(default_factory=list) + limit: int = 0 + user: str = "" + collection: str = "" -class DocumentEmbeddingsResponse(Record): - error = Error() - chunks = Array(String()) +@dataclass +class DocumentEmbeddingsResponse: + error: Error | None = None + chunks: list[str] = field(default_factory=list) document_embeddings_request_queue = topic( - "non-persistent://trustgraph/document-embeddings-request" + "document-embeddings-request", qos='q0', tenant='trustgraph', namespace='flow' ) document_embeddings_response_queue = topic( - "non-persistent://trustgraph/document-embeddings-response" + "document-embeddings-response", qos='q0', tenant='trustgraph', namespace='flow' ) \ No newline at end of file diff --git a/trustgraph-base/trustgraph/schema/services/retrieval.py b/trustgraph-base/trustgraph/schema/services/retrieval.py index 3cd7f792..813fbaa0 100644 --- a/trustgraph-base/trustgraph/schema/services/retrieval.py +++ b/trustgraph-base/trustgraph/schema/services/retrieval.py @@ -1,5 +1,4 @@ - -from pulsar.schema import Record, Bytes, String, Boolean, Integer, Array, Double +from dataclasses import dataclass from ..core.topic import topic from ..core.primitives import Error, Value @@ -7,36 +6,39 @@ from ..core.primitives import Error, Value # Graph RAG text retrieval -class GraphRagQuery(Record): - query = String() - user = String() - collection = String() - entity_limit = Integer() - triple_limit = Integer() - max_subgraph_size = Integer() - max_path_length = Integer() - streaming = Boolean() +@dataclass +class GraphRagQuery: + query: str = "" + user: str = "" + collection: str = "" + entity_limit: int = 0 + triple_limit: int = 0 + max_subgraph_size: int = 0 + max_path_length: int = 0 + streaming: bool = False -class GraphRagResponse(Record): - error = Error() - response = String() - chunk = String() - end_of_stream = Boolean() +@dataclass +class GraphRagResponse: + error: Error | None = None + response: str = "" + chunk: str = "" + end_of_stream: bool = False ############################################################################ # Document RAG text retrieval -class DocumentRagQuery(Record): - query = String() - user = String() - collection = String() - doc_limit = Integer() - streaming = Boolean() - -class DocumentRagResponse(Record): - error = Error() - response = String() - chunk = String() - end_of_stream = Boolean() +@dataclass +class DocumentRagQuery: + query: str = "" + user: str = "" + collection: str = "" + doc_limit: int = 0 + streaming: bool = False +@dataclass +class DocumentRagResponse: + error: Error | None = None + response: str = "" + chunk: str = "" + end_of_stream: bool = False diff --git a/trustgraph-base/trustgraph/schema/services/storage.py b/trustgraph-base/trustgraph/schema/services/storage.py index 16791615..e65fb793 100644 --- a/trustgraph-base/trustgraph/schema/services/storage.py +++ b/trustgraph-base/trustgraph/schema/services/storage.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String +from dataclasses import dataclass from ..core.primitives import Error from ..core.topic import topic @@ -7,15 +7,17 @@ from ..core.topic import topic # Storage management operations -class StorageManagementRequest(Record): +@dataclass +class StorageManagementRequest: """Request for storage management operations sent to store processors""" - operation = String() # e.g., "delete-collection" - user = String() - collection = String() + operation: str = "" # e.g., "delete-collection" + user: str = "" + collection: str = "" -class StorageManagementResponse(Record): +@dataclass +class StorageManagementResponse: """Response from storage processors for management operations""" - error = Error() # Only populated if there's an error, if null success + error: Error | None = None # Only populated if there's an error, if null success ############################################################################ @@ -23,20 +25,21 @@ class StorageManagementResponse(Record): # Topics for sending collection management requests to different storage types vector_storage_management_topic = topic( - 'vector-storage-management', kind='non-persistent', namespace='request' + 'vector-storage-management', qos='q0', namespace='request' ) object_storage_management_topic = topic( - 'object-storage-management', kind='non-persistent', namespace='request' + 'object-storage-management', qos='q0', namespace='request' ) triples_storage_management_topic = topic( - 'triples-storage-management', kind='non-persistent', namespace='request' + 'triples-storage-management', qos='q0', namespace='request' ) # Topic for receiving responses from storage processors storage_management_response_topic = topic( - 'storage-management', kind='non-persistent', namespace='response' + 'storage-management', qos='q0', namespace='response' ) ############################################################################ + diff --git a/trustgraph-base/trustgraph/schema/services/structured_query.py b/trustgraph-base/trustgraph/schema/services/structured_query.py index df21bfe2..ae1eaa5f 100644 --- a/trustgraph-base/trustgraph/schema/services/structured_query.py +++ b/trustgraph-base/trustgraph/schema/services/structured_query.py @@ -1,4 +1,4 @@ -from pulsar.schema import Record, String, Map, Array +from dataclasses import dataclass, field from ..core.primitives import Error from ..core.topic import topic @@ -7,14 +7,17 @@ from ..core.topic import topic # Structured Query Service - executes GraphQL queries -class StructuredQueryRequest(Record): - question = String() - user = String() # Cassandra keyspace identifier - collection = String() # Data collection identifier +@dataclass +class StructuredQueryRequest: + question: str = "" + user: str = "" # Cassandra keyspace identifier + collection: str = "" # Data collection identifier -class StructuredQueryResponse(Record): - error = Error() - data = String() # JSON-encoded GraphQL response data - errors = Array(String()) # GraphQL errors if any +@dataclass +class StructuredQueryResponse: + error: Error | None = None + data: str = "" # JSON-encoded GraphQL response data + errors: list[str] = field(default_factory=list) # GraphQL errors if any ############################################################################ +