diff --git a/specs/api/components/schemas/agent/AgentRequest.yaml b/specs/api/components/schemas/agent/AgentRequest.yaml new file mode 100644 index 00000000..ddf2019a --- /dev/null +++ b/specs/api/components/schemas/agent/AgentRequest.yaml @@ -0,0 +1,59 @@ +type: object +description: | + Agent service request - conversational AI agent that can reason and take actions. +required: + - question +properties: + question: + type: string + description: User question or prompt for the agent + example: What is the capital of France? + state: + type: string + description: Agent state for continuation (optional, for multi-turn) + example: agent-state-12345 + group: + type: array + description: Group identifiers for collaborative agents (optional) + items: + type: string + example: ["research-team"] + history: + type: array + description: Conversation history (optional, list of previous agent steps) + items: + type: object + properties: + thought: + type: string + description: Agent's reasoning + example: I need to search for information about Paris + action: + type: string + description: Action taken + example: search + arguments: + type: object + description: Action arguments + additionalProperties: + type: string + example: + query: "capital of France" + observation: + type: string + description: Result of the action + example: "Paris is the capital of France" + user: + type: string + description: User context for this step + example: alice + user: + type: string + description: User identifier for multi-tenancy + default: trustgraph + example: alice + streaming: + type: boolean + description: Enable streaming response delivery + default: false + example: true diff --git a/specs/api/components/schemas/agent/AgentResponse.yaml b/specs/api/components/schemas/agent/AgentResponse.yaml new file mode 100644 index 00000000..86d636b5 --- /dev/null +++ b/specs/api/components/schemas/agent/AgentResponse.yaml @@ -0,0 +1,51 @@ +type: object +description: Agent service response (streaming or legacy format) +properties: + chunk-type: + type: string + description: Type of streaming chunk (streaming mode only) + enum: + - thought + - action + - observation + - answer + - error + example: answer + content: + type: string + description: Chunk content (streaming mode only) + example: Paris is the capital of France. + end-of-message: + type: boolean + description: Current chunk type is complete (streaming mode) + default: false + example: true + end-of-dialog: + type: boolean + description: Entire agent dialog is complete (streaming mode) + default: false + example: true + answer: + type: string + description: Final answer (legacy non-streaming format) + example: Paris is the capital of France. + thought: + type: string + description: Agent reasoning (legacy format) + example: I should search for information about the capital of France. + observation: + type: string + description: Observation from actions (legacy format) + example: Found information about Paris being the capital. + error: + type: object + description: Error details if request failed + properties: + message: + type: string + description: Error message + example: Failed to process agent request + code: + type: string + description: Error code + example: AGENT_ERROR diff --git a/specs/api/components/schemas/collection/CollectionRequest.yaml b/specs/api/components/schemas/collection/CollectionRequest.yaml new file mode 100644 index 00000000..bf3ab7d4 --- /dev/null +++ b/specs/api/components/schemas/collection/CollectionRequest.yaml @@ -0,0 +1,58 @@ +type: object +description: | + Collection management request. + + Operations: list-collections, update-collection, delete-collection +required: + - operation +properties: + operation: + type: string + enum: + - list-collections + - update-collection + - delete-collection + description: | + Collection operation: + - `list-collections`: List collections for user + - `update-collection`: Create or update collection metadata + - `delete-collection`: Delete collection + user: + type: string + description: User identifier + default: trustgraph + example: alice + collection: + type: string + description: Collection identifier (for update, delete) + example: research + timestamp: + type: string + description: ISO timestamp + format: date-time + example: "2024-01-15T10:30:00Z" + name: + type: string + description: Human-readable collection name (for update) + example: Research Papers + description: + type: string + description: Collection description (for update) + example: Academic research papers on AI and ML + tags: + type: array + description: Collection tags for organization (for update) + items: + type: string + example: ["research", "AI", "academic"] + tag-filter: + type: array + description: Filter collections by tags (for list) + items: + type: string + example: ["research"] + limit: + type: integer + description: Maximum number of results (for list) + default: 0 + example: 100 diff --git a/specs/api/components/schemas/collection/CollectionResponse.yaml b/specs/api/components/schemas/collection/CollectionResponse.yaml new file mode 100644 index 00000000..f924cbf5 --- /dev/null +++ b/specs/api/components/schemas/collection/CollectionResponse.yaml @@ -0,0 +1,39 @@ +type: object +description: Collection management response +properties: + timestamp: + type: string + description: ISO timestamp + format: date-time + example: "2024-01-15T10:30:00Z" + collections: + type: array + description: List of collections (returned by list-collections) + items: + type: object + required: + - user + - collection + properties: + user: + type: string + description: User identifier + example: alice + collection: + type: string + description: Collection identifier + example: research + name: + type: string + description: Human-readable collection name + example: Research Papers + description: + type: string + description: Collection description + example: Academic research papers on AI and ML + tags: + type: array + description: Collection tags + items: + type: string + example: ["research", "AI", "academic"] diff --git a/specs/api/components/schemas/knowledge/KnowledgeRequest.yaml b/specs/api/components/schemas/knowledge/KnowledgeRequest.yaml new file mode 100644 index 00000000..5c40e118 --- /dev/null +++ b/specs/api/components/schemas/knowledge/KnowledgeRequest.yaml @@ -0,0 +1,128 @@ +type: object +description: | + Knowledge graph core management request. + + Operations: list-kg-cores, get-kg-core, put-kg-core, delete-kg-core, + load-kg-core, unload-kg-core +required: + - operation +properties: + operation: + type: string + enum: + - list-kg-cores + - get-kg-core + - put-kg-core + - delete-kg-core + - load-kg-core + - unload-kg-core + description: | + Knowledge core operation: + - `list-kg-cores`: List knowledge cores for user + - `get-kg-core`: Get knowledge core by ID + - `put-kg-core`: Store triples and/or embeddings + - `delete-kg-core`: Delete knowledge core by ID + - `load-kg-core`: Load knowledge core into flow + - `unload-kg-core`: Unload knowledge core from flow + user: + type: string + description: User identifier (for list-kg-cores, put-kg-core, delete-kg-core) + default: trustgraph + example: alice + id: + type: string + description: Knowledge core ID (for get, put, delete, load, unload) + example: core-123 + flow: + type: string + description: Flow ID (for load-kg-core) + example: my-flow + collection: + type: string + description: Collection identifier (for load-kg-core) + default: default + example: default + triples: + type: object + description: Triples to store (for put-kg-core) + required: + - metadata + - triples + properties: + metadata: + type: object + required: + - id + - user + - collection + properties: + id: + type: string + description: Knowledge core ID + example: core-123 + user: + type: string + description: User identifier + example: alice + collection: + type: string + description: Collection identifier + example: default + metadata: + type: array + description: Metadata triples + items: + $ref: '../../common/Triple.yaml' + triples: + type: array + description: Knowledge triples + items: + $ref: '../../common/Triple.yaml' + graph-embeddings: + type: object + description: Graph embeddings to store (for put-kg-core) + required: + - metadata + - entities + properties: + metadata: + type: object + required: + - id + - user + - collection + properties: + id: + type: string + description: Knowledge core ID + example: core-123 + user: + type: string + description: User identifier + example: alice + collection: + type: string + description: Collection identifier + example: default + metadata: + type: array + description: Metadata triples + items: + $ref: '../../common/Triple.yaml' + entities: + type: array + description: Entity embeddings + items: + type: object + required: + - entity + - vectors + properties: + entity: + $ref: '../../common/RdfValue.yaml' + vectors: + type: array + description: Embedding vectors + items: + type: number + example: [0.1, 0.2, 0.3] diff --git a/specs/api/components/schemas/knowledge/KnowledgeResponse.yaml b/specs/api/components/schemas/knowledge/KnowledgeResponse.yaml new file mode 100644 index 00000000..229233ca --- /dev/null +++ b/specs/api/components/schemas/knowledge/KnowledgeResponse.yaml @@ -0,0 +1,91 @@ +type: object +description: Knowledge service response +properties: + ids: + type: array + description: List of knowledge core IDs (returned by list-kg-cores) + items: + type: string + example: ["core-123", "core-456"] + triples: + type: object + description: Triples data (returned by get-kg-core, streamed) + properties: + metadata: + type: object + required: + - id + - user + - collection + properties: + id: + type: string + description: Knowledge core ID + example: core-123 + user: + type: string + description: User identifier + example: alice + collection: + type: string + description: Collection identifier + example: default + metadata: + type: array + description: Metadata triples + items: + $ref: '../../common/Triple.yaml' + triples: + type: array + description: Knowledge triples + items: + $ref: '../../common/Triple.yaml' + graph-embeddings: + type: object + description: Graph embeddings data (returned by get-kg-core, streamed) + properties: + metadata: + type: object + required: + - id + - user + - collection + properties: + id: + type: string + description: Knowledge core ID + example: core-123 + user: + type: string + description: User identifier + example: alice + collection: + type: string + description: Collection identifier + example: default + metadata: + type: array + description: Metadata triples + items: + $ref: '../../common/Triple.yaml' + entities: + type: array + description: Entity embeddings + items: + type: object + required: + - entity + - vectors + properties: + entity: + $ref: '../../common/RdfValue.yaml' + vectors: + type: array + description: Embedding vectors + items: + type: number + example: [0.1, 0.2, 0.3] + eos: + type: boolean + description: End of stream marker (for streaming responses) + example: true diff --git a/specs/api/components/schemas/librarian/LibrarianRequest.yaml b/specs/api/components/schemas/librarian/LibrarianRequest.yaml new file mode 100644 index 00000000..18aa94b1 --- /dev/null +++ b/specs/api/components/schemas/librarian/LibrarianRequest.yaml @@ -0,0 +1,79 @@ +type: object +description: | + Librarian service request for document library management. + + Operations: add-document, remove-document, list-documents, + start-processing, stop-processing, list-processing +required: + - operation +properties: + operation: + type: string + enum: + - add-document + - remove-document + - list-documents + - start-processing + - stop-processing + - list-processing + description: | + Library operation: + - `add-document`: Add document to library + - `remove-document`: Remove document from library + - `list-documents`: List documents in library + - `start-processing`: Start processing library documents + - `stop-processing`: Stop library processing + - `list-processing`: List processing status + flow: + type: string + description: Flow ID + example: my-flow + collection: + type: string + description: Collection identifier + default: default + example: default + user: + type: string + description: User identifier + default: trustgraph + example: alice + document-id: + type: string + description: Document identifier + example: doc-123 + processing-id: + type: string + description: Processing task identifier + example: proc-456 + document-metadata: + $ref: '../common/DocumentMetadata.yaml' + processing-metadata: + $ref: '../common/ProcessingMetadata.yaml' + content: + type: string + description: Document content (for add-document with inline content) + example: This is the document content... + criteria: + type: array + description: Search criteria for filtering documents + items: + type: object + required: + - key + - value + - operator + properties: + key: + type: string + description: Metadata field name + example: author + value: + type: string + description: Value to match + example: John Doe + operator: + type: string + enum: [eq, ne, gt, lt, contains] + description: Comparison operator + example: eq diff --git a/specs/api/components/schemas/librarian/LibrarianResponse.yaml b/specs/api/components/schemas/librarian/LibrarianResponse.yaml new file mode 100644 index 00000000..caa84628 --- /dev/null +++ b/specs/api/components/schemas/librarian/LibrarianResponse.yaml @@ -0,0 +1,18 @@ +type: object +description: Librarian service response +properties: + document-metadata: + $ref: '../common/DocumentMetadata.yaml' + content: + type: string + description: Document content + document-metadatas: + type: array + description: List of documents (returned by list-documents) + items: + $ref: '../common/DocumentMetadata.yaml' + processing-metadatas: + type: array + description: List of processing tasks (returned by list-processing) + items: + $ref: '../common/ProcessingMetadata.yaml' diff --git a/specs/api/components/schemas/rag/DocumentRagRequest.yaml b/specs/api/components/schemas/rag/DocumentRagRequest.yaml new file mode 100644 index 00000000..97a9d2ff --- /dev/null +++ b/specs/api/components/schemas/rag/DocumentRagRequest.yaml @@ -0,0 +1,33 @@ +type: object +description: | + Document RAG (Retrieval-Augmented Generation) query request. + Searches document embeddings and generates answer using retrieved context. +required: + - query +properties: + query: + type: string + description: User query or question + example: What are the key findings in the research papers? + user: + type: string + description: User identifier for multi-tenancy + default: trustgraph + example: alice + collection: + type: string + description: Collection to search within + default: default + example: research + doc-limit: + type: integer + description: Maximum number of documents to retrieve + default: 20 + minimum: 1 + maximum: 100 + example: 10 + streaming: + type: boolean + description: Enable streaming response delivery + default: false + example: true diff --git a/specs/api/components/schemas/rag/DocumentRagResponse.yaml b/specs/api/components/schemas/rag/DocumentRagResponse.yaml new file mode 100644 index 00000000..6a0166e7 --- /dev/null +++ b/specs/api/components/schemas/rag/DocumentRagResponse.yaml @@ -0,0 +1,24 @@ +type: object +description: Document RAG response +properties: + response: + type: string + description: Generated response based on retrieved documents + example: The research papers found three key findings... + end-of-stream: + type: boolean + description: Indicates streaming is complete (streaming mode) + default: false + example: true + error: + type: object + description: Error details if request failed + properties: + message: + type: string + description: Error message + example: Failed to retrieve documents + type: + type: string + description: Error type + example: RETRIEVAL_ERROR diff --git a/specs/api/components/schemas/rag/GraphRagRequest.yaml b/specs/api/components/schemas/rag/GraphRagRequest.yaml new file mode 100644 index 00000000..733dd7c1 --- /dev/null +++ b/specs/api/components/schemas/rag/GraphRagRequest.yaml @@ -0,0 +1,54 @@ +type: object +description: | + Graph RAG (Retrieval-Augmented Generation) query request. + Searches knowledge graph and generates answer using retrieved subgraph. +required: + - query +properties: + query: + type: string + description: User query or question + example: What connections exist between quantum physics and computer science? + user: + type: string + description: User identifier for multi-tenancy + default: trustgraph + example: alice + collection: + type: string + description: Collection to search within + default: default + example: research + entity-limit: + type: integer + description: Maximum number of entities to retrieve + default: 50 + minimum: 1 + maximum: 200 + example: 30 + triple-limit: + type: integer + description: Maximum number of triples to retrieve per entity + default: 30 + minimum: 1 + maximum: 100 + example: 20 + max-subgraph-size: + type: integer + description: Maximum total subgraph size (triples) + default: 1000 + minimum: 10 + maximum: 5000 + example: 500 + max-path-length: + type: integer + description: Maximum path length for graph traversal + default: 2 + minimum: 1 + maximum: 5 + example: 3 + streaming: + type: boolean + description: Enable streaming response delivery + default: false + example: true diff --git a/specs/api/components/schemas/rag/GraphRagResponse.yaml b/specs/api/components/schemas/rag/GraphRagResponse.yaml new file mode 100644 index 00000000..75f4f059 --- /dev/null +++ b/specs/api/components/schemas/rag/GraphRagResponse.yaml @@ -0,0 +1,24 @@ +type: object +description: Graph RAG response +properties: + response: + type: string + description: Generated response based on retrieved knowledge graph + example: Quantum physics and computer science intersect in quantum computing... + end-of-stream: + type: boolean + description: Indicates streaming is complete (streaming mode) + default: false + example: true + error: + type: object + description: Error details if request failed + properties: + message: + type: string + description: Error message + example: Failed to retrieve graph data + type: + type: string + description: Error type + example: GRAPH_ERROR diff --git a/specs/api/components/schemas/text-completion/TextCompletionRequest.yaml b/specs/api/components/schemas/text-completion/TextCompletionRequest.yaml new file mode 100644 index 00000000..95c5a30d --- /dev/null +++ b/specs/api/components/schemas/text-completion/TextCompletionRequest.yaml @@ -0,0 +1,20 @@ +type: object +description: | + Text completion request - direct LLM completion without RAG. +required: + - system + - prompt +properties: + system: + type: string + description: System prompt that sets behavior and context for the LLM + example: You are a helpful assistant that provides concise answers. + prompt: + type: string + description: User prompt or question + example: Explain the concept of recursion in programming. + streaming: + type: boolean + description: Enable streaming response delivery + default: false + example: true diff --git a/specs/api/components/schemas/text-completion/TextCompletionResponse.yaml b/specs/api/components/schemas/text-completion/TextCompletionResponse.yaml new file mode 100644 index 00000000..b97573c7 --- /dev/null +++ b/specs/api/components/schemas/text-completion/TextCompletionResponse.yaml @@ -0,0 +1,26 @@ +type: object +description: Text completion response +required: + - response +properties: + response: + type: string + description: Generated text response + example: Recursion is a programming technique where a function calls itself... + in-token: + type: integer + description: Number of input tokens consumed + example: 45 + out-token: + type: integer + description: Number of output tokens generated + example: 128 + model: + type: string + description: Model used for completion + example: gpt-4 + end-of-stream: + type: boolean + description: Indicates streaming is complete (streaming mode) + default: false + example: true diff --git a/specs/api/openapi.yaml b/specs/api/openapi.yaml index 4398665c..b6f57454 100644 --- a/specs/api/openapi.yaml +++ b/specs/api/openapi.yaml @@ -99,6 +99,22 @@ paths: $ref: './paths/config.yaml' /api/v1/flow: $ref: './paths/flow.yaml' + /api/v1/librarian: + $ref: './paths/librarian.yaml' + /api/v1/knowledge: + $ref: './paths/knowledge.yaml' + /api/v1/collection-management: + $ref: './paths/collection-management.yaml' + + # Flow-hosted services (require running flow instance) + /api/v1/flow/{flow}/service/agent: + $ref: './paths/flow/agent.yaml' + /api/v1/flow/{flow}/service/document-rag: + $ref: './paths/flow/document-rag.yaml' + /api/v1/flow/{flow}/service/graph-rag: + $ref: './paths/flow/graph-rag.yaml' + /api/v1/flow/{flow}/service/text-completion: + $ref: './paths/flow/text-completion.yaml' # More services will be added here as we build them components: diff --git a/specs/api/paths/collection-management.yaml b/specs/api/paths/collection-management.yaml new file mode 100644 index 00000000..7dffd4e0 --- /dev/null +++ b/specs/api/paths/collection-management.yaml @@ -0,0 +1,108 @@ +post: + tags: + - Collection + summary: Collection metadata management + description: | + Manage collection metadata for organizing documents and knowledge. + + ## Collections + + Collections are organizational units for grouping: + - Documents in the librarian + - Knowledge cores + - User data + + Each collection has: + - **user**: Owner identifier + - **collection**: Unique collection ID + - **name**: Human-readable display name + - **description**: Purpose and contents + - **tags**: Labels for filtering and organization + + ## Operations + + ### list-collections + List all collections for a user. Optionally filter by tags and limit results. + Returns array of collection metadata. + + ### update-collection + Create or update collection metadata. If collection doesn't exist, it's created. + If it exists, metadata is updated. Allows setting name, description, and tags. + + ### delete-collection + Delete a collection by user and collection ID. This removes the metadata but + typically does not delete the associated data (documents, knowledge cores). + + operationId: collectionManagementService + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/schemas/collection/CollectionRequest.yaml' + examples: + listCollections: + summary: List all collections for user + value: + operation: list-collections + user: alice + listCollectionsFiltered: + summary: List collections filtered by tags + value: + operation: list-collections + user: alice + tag-filter: ["research", "AI"] + limit: 50 + updateCollection: + summary: Create/update collection + value: + operation: update-collection + user: alice + collection: research + name: Research Papers + description: Academic research papers on AI and ML + tags: ["research", "AI", "academic"] + timestamp: "2024-01-15T10:30:00Z" + deleteCollection: + summary: Delete collection + value: + operation: delete-collection + user: alice + collection: research + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../components/schemas/collection/CollectionResponse.yaml' + examples: + listCollections: + summary: List of collections + value: + timestamp: "2024-01-15T10:30:00Z" + collections: + - user: alice + collection: research + name: Research Papers + description: Academic research papers on AI and ML + tags: ["research", "AI", "academic"] + - user: alice + collection: personal + name: Personal Documents + description: Personal notes and documents + tags: ["personal"] + updateSuccess: + summary: Update successful + value: + timestamp: "2024-01-15T10:30:00Z" + deleteSuccess: + summary: Delete successful + value: + timestamp: "2024-01-15T10:30:00Z" + '401': + $ref: '../components/responses/Unauthorized.yaml' + '500': + $ref: '../components/responses/Error.yaml' diff --git a/specs/api/paths/flow/agent.yaml b/specs/api/paths/flow/agent.yaml new file mode 100644 index 00000000..697f9b64 --- /dev/null +++ b/specs/api/paths/flow/agent.yaml @@ -0,0 +1,130 @@ +post: + tags: + - AI Services + summary: Agent service - conversational AI with reasoning + description: | + AI agent that can understand questions, reason about them, and take actions. + + ## Agent Overview + + The agent service provides a conversational AI that: + - Understands natural language questions + - Reasons about problems using thoughts + - Takes actions to gather information + - Provides coherent answers + + ## Request Format + + Send a question with optional: + - **state**: Continue from previous conversation + - **history**: Previous agent steps for context + - **group**: Collaborative agent identifiers + - **streaming**: Enable streaming responses + + ## Response Modes + + ### Streaming Mode (streaming: true) + Responses arrive as chunks with `chunk-type`: + - `thought`: Agent's reasoning process + - `action`: Action being taken + - `observation`: Result from action + - `answer`: Final response to user + - `error`: Error occurred + + Each chunk may have multiple messages. Check flags: + - `end-of-message`: Current chunk type complete + - `end-of-dialog`: Entire conversation complete + + ### Legacy Mode (streaming: false) + Single response with: + - `answer`: Complete answer + - `thought`: Reasoning (if any) + - `observation`: Observations (if any) + + ## Multi-turn Conversations + + Include `history` array with previous steps to maintain context. + Each step has: thought, action, arguments, observation. + + operationId: agentService + security: + - bearerAuth: [] + parameters: + - name: flow + in: path + required: true + schema: + type: string + description: Flow instance ID + example: my-flow + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../components/schemas/agent/AgentRequest.yaml' + examples: + simpleQuestion: + summary: Simple question + value: + question: What is the capital of France? + user: alice + streamingQuestion: + summary: Question with streaming enabled + value: + question: Explain quantum computing + user: alice + streaming: true + conversationWithHistory: + summary: Multi-turn conversation + value: + question: And what about its population? + user: alice + history: + - thought: User is asking about the capital of France + action: search + arguments: + query: "capital of France" + observation: "Paris is the capital of France" + user: alice + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../../components/schemas/agent/AgentResponse.yaml' + examples: + streamingThought: + summary: Streaming thought chunk + value: + chunk-type: thought + content: I need to search for information about quantum computing + end-of-message: false + end-of-dialog: false + streamingAnswer: + summary: Streaming answer chunk + value: + chunk-type: answer + content: Quantum computing uses quantum mechanics principles... + end-of-message: false + end-of-dialog: false + streamingComplete: + summary: Streaming complete marker + value: + chunk-type: answer + content: "" + end-of-message: true + end-of-dialog: true + legacyResponse: + summary: Legacy non-streaming response + value: + answer: Paris is the capital of France. + thought: User is asking about the capital of France + observation: "" + end-of-message: false + end-of-dialog: false + '401': + $ref: '../../components/responses/Unauthorized.yaml' + '500': + $ref: '../../components/responses/Error.yaml' diff --git a/specs/api/paths/flow/document-rag.yaml b/specs/api/paths/flow/document-rag.yaml new file mode 100644 index 00000000..6546234a --- /dev/null +++ b/specs/api/paths/flow/document-rag.yaml @@ -0,0 +1,107 @@ +post: + tags: + - AI Services + summary: Document RAG - retrieve and generate from documents + description: | + Retrieval-Augmented Generation over document embeddings. + + ## Document RAG Overview + + Document RAG combines: + 1. **Retrieval**: Search document embeddings using semantic similarity + 2. **Generation**: Use LLM to synthesize answer from retrieved documents + + This provides grounded answers based on your document corpus. + + ## Query Process + + 1. Convert query to embedding + 2. Search document embeddings for most similar chunks + 3. Retrieve top N document chunks (configurable via doc-limit) + 4. Pass query + retrieved context to LLM + 5. Generate answer grounded in documents + + ## Streaming + + Enable `streaming: true` to receive the answer as it's generated: + - Multiple messages with `response` content + - Final message with `end-of-stream: true` + + Without streaming, returns complete answer in single response. + + ## Parameters + + - **doc-limit**: Controls retrieval depth (1-100, default 20) + - Higher = more context but slower + - Lower = faster but may miss relevant info + - **collection**: Target specific document collection + - **user**: Multi-tenant isolation + + operationId: documentRagService + security: + - bearerAuth: [] + parameters: + - name: flow + in: path + required: true + schema: + type: string + description: Flow instance ID + example: my-flow + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../components/schemas/rag/DocumentRagRequest.yaml' + examples: + basicQuery: + summary: Basic document query + value: + query: What are the key findings in the research papers? + user: alice + collection: research + streamingQuery: + summary: Streaming query + value: + query: Summarize the main conclusions + user: alice + collection: research + doc-limit: 15 + streaming: true + limitedRetrieval: + summary: Query with limited retrieval + value: + query: What is quantum entanglement? + doc-limit: 5 + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../../components/schemas/rag/DocumentRagResponse.yaml' + examples: + completeResponse: + summary: Complete non-streaming response + value: + response: | + The research papers present three key findings: + 1. Quantum entanglement exhibits non-local correlations + 2. Bell's inequality is violated in experimental tests + 3. Applications in quantum cryptography are promising + end-of-stream: false + streamingChunk: + summary: Streaming response chunk + value: + response: "The research papers present three" + end-of-stream: false + streamingComplete: + summary: Streaming complete marker + value: + response: "" + end-of-stream: true + '401': + $ref: '../../components/responses/Unauthorized.yaml' + '500': + $ref: '../../components/responses/Error.yaml' diff --git a/specs/api/paths/flow/graph-rag.yaml b/specs/api/paths/flow/graph-rag.yaml new file mode 100644 index 00000000..1de94406 --- /dev/null +++ b/specs/api/paths/flow/graph-rag.yaml @@ -0,0 +1,127 @@ +post: + tags: + - AI Services + summary: Graph RAG - retrieve and generate from knowledge graph + description: | + Retrieval-Augmented Generation over knowledge graph. + + ## Graph RAG Overview + + Graph RAG combines: + 1. **Retrieval**: Find relevant entities and subgraph from knowledge graph + 2. **Generation**: Use LLM to reason over graph structure and generate answer + + This provides graph-aware answers that leverage relationships and structure. + + ## Query Process + + 1. Identify relevant entities from query (using embeddings) + 2. Retrieve connected subgraph around entities + 3. Optionally traverse paths up to max-path-length hops + 4. Limit subgraph size to stay within context window + 5. Pass query + graph structure to LLM + 6. Generate answer incorporating graph relationships + + ## Streaming + + Enable `streaming: true` to receive the answer as it's generated: + - Multiple messages with `response` content + - Final message with `end-of-stream: true` + + Without streaming, returns complete answer in single response. + + ## Parameters + + Control retrieval scope with multiple knobs: + - **entity-limit**: How many starting entities to find (1-200, default 50) + - **triple-limit**: Triples per entity (1-100, default 30) + - **max-subgraph-size**: Total subgraph cap (10-5000, default 1000) + - **max-path-length**: Graph traversal depth (1-5, default 2) + + Higher limits = more context but: + - Slower retrieval + - Larger context for LLM + - May hit context window limits + + ## Use Cases + + Best for queries requiring: + - Relationship understanding ("How are X and Y connected?") + - Multi-hop reasoning ("What's the path from A to B?") + - Structural analysis ("What are the main entities related to X?") + + operationId: graphRagService + security: + - bearerAuth: [] + parameters: + - name: flow + in: path + required: true + schema: + type: string + description: Flow instance ID + example: my-flow + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../components/schemas/rag/GraphRagRequest.yaml' + examples: + basicQuery: + summary: Basic graph query + value: + query: What connections exist between quantum physics and computer science? + user: alice + collection: research + streamingQuery: + summary: Streaming query with custom limits + value: + query: Trace the historical development of AI from Turing to modern LLMs + user: alice + collection: research + entity-limit: 40 + triple-limit: 25 + max-subgraph-size: 800 + max-path-length: 3 + streaming: true + focusedQuery: + summary: Focused query with tight limits + value: + query: What is the immediate relationship between entity A and B? + entity-limit: 10 + triple-limit: 15 + max-subgraph-size: 200 + max-path-length: 1 + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../../components/schemas/rag/GraphRagResponse.yaml' + examples: + completeResponse: + summary: Complete non-streaming response + value: + response: | + Quantum physics and computer science intersect primarily through quantum computing. + The knowledge graph shows connections through: + - Quantum algorithms (Shor's algorithm, Grover's algorithm) + - Quantum information theory + - Computational complexity theory + end-of-stream: false + streamingChunk: + summary: Streaming response chunk + value: + response: "Quantum physics and computer science intersect" + end-of-stream: false + streamingComplete: + summary: Streaming complete marker + value: + response: "" + end-of-stream: true + '401': + $ref: '../../components/responses/Unauthorized.yaml' + '500': + $ref: '../../components/responses/Error.yaml' diff --git a/specs/api/paths/flow/text-completion.yaml b/specs/api/paths/flow/text-completion.yaml new file mode 100644 index 00000000..4a011827 --- /dev/null +++ b/specs/api/paths/flow/text-completion.yaml @@ -0,0 +1,125 @@ +post: + tags: + - AI Services + summary: Text completion - direct LLM generation + description: | + Direct text completion using LLM without retrieval augmentation. + + ## Text Completion Overview + + Pure LLM generation for: + - General knowledge questions + - Creative writing + - Code generation + - Analysis and reasoning + - Any task not requiring specific document/graph context + + ## System vs Prompt + + - **system**: Sets LLM behavior, role, constraints + - "You are a helpful assistant" + - "You are an expert Python developer" + - "Respond in JSON format" + - **prompt**: The actual user request/question + + ## Streaming + + Enable `streaming: true` to receive tokens as generated: + - Multiple messages with partial `response` + - Final message with `end-of-stream: true` + + Without streaming, returns complete response in single message. + + ## Token Counting + + Response includes token usage: + - `in-token`: Input tokens (system + prompt) + - `out-token`: Generated tokens + - Useful for cost tracking and optimization + + ## When to Use + + Use text-completion when: + - No specific context needed (general knowledge) + - System prompt provides sufficient context + - Want direct control over prompting + + Use document-rag/graph-rag when: + - Need to ground response in specific documents + - Want to leverage knowledge graph relationships + - Require citations or provenance + + operationId: textCompletionService + security: + - bearerAuth: [] + parameters: + - name: flow + in: path + required: true + schema: + type: string + description: Flow instance ID + example: my-flow + requestBody: + required: true + content: + application/json: + schema: + $ref: '../../components/schemas/text-completion/TextCompletionRequest.yaml' + examples: + basicCompletion: + summary: Basic text completion + value: + system: You are a helpful assistant that provides concise answers. + prompt: Explain the concept of recursion in programming. + codeGeneration: + summary: Code generation with streaming + value: + system: You are an expert Python developer. Provide clean, well-documented code. + prompt: Write a function to calculate the Fibonacci sequence using memoization. + streaming: true + jsonResponse: + summary: Structured output request + value: + system: You are a JSON API. Respond only with valid JSON, no other text. + prompt: | + Extract key information from this text and return as JSON with fields: + title, author, year, summary. + + Text: "The Theory of Everything by Stephen Hawking (2006) explores..." + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../../components/schemas/text-completion/TextCompletionResponse.yaml' + examples: + completeResponse: + summary: Complete non-streaming response + value: + response: | + Recursion is a programming technique where a function calls itself + to solve a problem by breaking it down into smaller, similar subproblems. + Each recursive call works on a simpler version until reaching a base case. + in-token: 45 + out-token: 128 + model: gpt-4 + end-of-stream: false + streamingChunk: + summary: Streaming response chunk + value: + response: "Recursion is a programming technique" + end-of-stream: false + streamingComplete: + summary: Streaming complete with tokens + value: + response: "" + in-token: 45 + out-token: 128 + model: gpt-4 + end-of-stream: true + '401': + $ref: '../../components/responses/Unauthorized.yaml' + '500': + $ref: '../../components/responses/Error.yaml' diff --git a/specs/api/paths/knowledge.yaml b/specs/api/paths/knowledge.yaml new file mode 100644 index 00000000..71bba496 --- /dev/null +++ b/specs/api/paths/knowledge.yaml @@ -0,0 +1,196 @@ +post: + tags: + - Knowledge + summary: Knowledge graph core management + description: | + Manage knowledge graph cores - persistent storage of triples and embeddings. + + ## Knowledge Cores + + Knowledge cores are the foundational storage units for: + - **Triples**: RDF triples representing knowledge graph data + - **Graph Embeddings**: Vector embeddings for entities + - **Metadata**: Descriptive information about the knowledge + + Each core has an ID, user, and collection for organization. + + ## Operations + + ### list-kg-cores + List all knowledge cores for a user. Returns array of core IDs. + + ### get-kg-core + Retrieve a knowledge core by ID. Returns triples and/or graph embeddings. + Response is streamed - may receive multiple messages followed by EOS marker. + + ### put-kg-core + Store triples and/or graph embeddings. Creates new core or updates existing. + Can store triples only, embeddings only, or both together. + + ### delete-kg-core + Delete a knowledge core by ID. Removes all associated data. + + ### load-kg-core + Load a knowledge core into a running flow's collection. + Makes the data available for querying within that flow instance. + + ### unload-kg-core + Unload a knowledge core from a flow's collection. + Removes data from flow instance but doesn't delete the core. + + ## Streaming Responses + + The `get-kg-core` operation streams data in chunks: + 1. Multiple messages with `triples` or `graph-embeddings` + 2. Final message with `eos: true` to signal completion + + operationId: knowledgeService + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/schemas/knowledge/KnowledgeRequest.yaml' + examples: + listKnowledgeCores: + summary: List knowledge cores + value: + operation: list-kg-cores + user: alice + getKnowledgeCore: + summary: Get knowledge core + value: + operation: get-kg-core + id: core-123 + putTriplesOnly: + summary: Store triples + value: + operation: put-kg-core + triples: + metadata: + id: core-123 + user: alice + collection: default + metadata: + - s: {v: "https://example.com/core-123", e: true} + p: {v: "https://www.w3.org/1999/02/22-rdf-syntax-ns#type", e: true} + o: {v: "https://trustgraph.ai/e/knowledge-core", e: true} + triples: + - s: {v: "https://example.com/entity1", e: true} + p: {v: "https://www.w3.org/2000/01/rdf-schema#label", e: true} + o: {v: "Entity 1", e: false} + - s: {v: "https://example.com/entity1", e: true} + p: {v: "https://example.com/relatedTo", e: true} + o: {v: "https://example.com/entity2", e: true} + putEmbeddingsOnly: + summary: Store embeddings + value: + operation: put-kg-core + graph-embeddings: + metadata: + id: core-123 + user: alice + collection: default + metadata: [] + entities: + - entity: {v: "https://example.com/entity1", e: true} + vectors: [0.1, 0.2, 0.3, 0.4, 0.5] + - entity: {v: "https://example.com/entity2", e: true} + vectors: [0.6, 0.7, 0.8, 0.9, 1.0] + putTriplesAndEmbeddings: + summary: Store triples and embeddings together + value: + operation: put-kg-core + triples: + metadata: + id: core-456 + user: bob + collection: research + metadata: [] + triples: + - s: {v: "https://example.com/doc1", e: true} + p: {v: "http://purl.org/dc/terms/title", e: true} + o: {v: "Research Paper", e: false} + graph-embeddings: + metadata: + id: core-456 + user: bob + collection: research + metadata: [] + entities: + - entity: {v: "https://example.com/doc1", e: true} + vectors: [0.11, 0.22, 0.33] + deleteKnowledgeCore: + summary: Delete knowledge core + value: + operation: delete-kg-core + id: core-123 + user: alice + loadKnowledgeCore: + summary: Load core into flow + value: + operation: load-kg-core + id: core-123 + flow: my-flow + collection: default + unloadKnowledgeCore: + summary: Unload core from flow + value: + operation: unload-kg-core + id: core-123 + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../components/schemas/knowledge/KnowledgeResponse.yaml' + examples: + listKnowledgeCores: + summary: List of knowledge cores + value: + ids: + - core-123 + - core-456 + - core-789 + getKnowledgeCoreTriples: + summary: Knowledge core triples (streaming) + value: + triples: + metadata: + id: core-123 + user: alice + collection: default + metadata: + - s: {v: "https://example.com/core-123", e: true} + p: {v: "https://www.w3.org/1999/02/22-rdf-syntax-ns#type", e: true} + o: {v: "https://trustgraph.ai/e/knowledge-core", e: true} + triples: + - s: {v: "https://example.com/entity1", e: true} + p: {v: "https://www.w3.org/2000/01/rdf-schema#label", e: true} + o: {v: "Entity 1", e: false} + getKnowledgeCoreEmbeddings: + summary: Knowledge core embeddings (streaming) + value: + graph-embeddings: + metadata: + id: core-123 + user: alice + collection: default + metadata: [] + entities: + - entity: {v: "https://example.com/entity1", e: true} + vectors: [0.1, 0.2, 0.3, 0.4, 0.5] + endOfStream: + summary: End of stream marker + value: + eos: true + deleteSuccess: + summary: Delete successful (empty response) + value: {} + '401': + $ref: '../components/responses/Unauthorized.yaml' + '500': + $ref: '../components/responses/Error.yaml' diff --git a/specs/api/paths/librarian.yaml b/specs/api/paths/librarian.yaml new file mode 100644 index 00000000..ffbc6d9c --- /dev/null +++ b/specs/api/paths/librarian.yaml @@ -0,0 +1,153 @@ +post: + tags: + - Librarian + summary: Document library management + description: | + Manage document library: add, remove, list documents, and control processing. + + ## Document Library + + The librarian service manages a persistent library of documents that can be: + - Added with metadata for organization + - Queried and filtered by criteria + - Processed through flows on-demand or continuously + - Tracked for processing status + + ## Operations + + ### add-document + Add a document to the library with metadata (URL, title, author, etc.). + Documents can be added by URL or with inline content. + + ### remove-document + Remove a document from the library by document ID or URL. + + ### list-documents + List all documents in the library, optionally filtered by criteria. + + ### start-processing + Start processing library documents through a flow. Documents are queued + for processing and handled asynchronously. + + ### stop-processing + Stop ongoing library document processing. + + ### list-processing + List current processing tasks and their status. + + operationId: librarianService + security: + - bearerAuth: [] + requestBody: + required: true + content: + application/json: + schema: + $ref: '../components/schemas/librarian/LibrarianRequest.yaml' + examples: + addDocumentByUrl: + summary: Add document by URL + value: + operation: add-document + flow: my-flow + collection: default + document-metadata: + url: https://example.com/document.pdf + title: Example Document + author: John Doe + metadata: + department: Engineering + category: Technical + addDocumentInline: + summary: Add document with inline content + value: + operation: add-document + flow: my-flow + collection: default + content: "This is the document content..." + document-metadata: + title: Inline Document + author: Jane Smith + removeDocument: + summary: Remove document + value: + operation: remove-document + flow: my-flow + collection: default + document-metadata: + url: https://example.com/document.pdf + listDocuments: + summary: List all documents + value: + operation: list-documents + flow: my-flow + collection: default + listDocumentsFiltered: + summary: List documents with criteria + value: + operation: list-documents + flow: my-flow + collection: default + criteria: + - key: author + value: John Doe + operator: eq + - key: department + value: Engineering + operator: eq + startProcessing: + summary: Start processing library documents + value: + operation: start-processing + flow: my-flow + collection: default + stopProcessing: + summary: Stop processing + value: + operation: stop-processing + flow: my-flow + collection: default + listProcessing: + summary: List processing status + value: + operation: list-processing + flow: my-flow + collection: default + responses: + '200': + description: Successful response + content: + application/json: + schema: + $ref: '../components/schemas/librarian/LibrarianResponse.yaml' + examples: + listDocuments: + summary: List of documents + value: + document-metadatas: + - url: https://example.com/doc1.pdf + title: Document 1 + author: John Doe + metadata: + department: Engineering + - url: https://example.com/doc2.pdf + title: Document 2 + author: Jane Smith + metadata: + department: Research + listProcessing: + summary: Processing status + value: + processing-metadatas: + - flow: my-flow + collection: default + status: processing + timestamp: "2024-01-15T10:30:00Z" + - flow: my-flow + collection: default + status: completed + timestamp: "2024-01-15T10:25:00Z" + '401': + $ref: '../components/responses/Unauthorized.yaml' + '500': + $ref: '../components/responses/Error.yaml'