feat: pluggable image-to-text service with OpenAI vision backend (#1038)

Adds a full-stack image description service: schema, base class, OpenAI backend, gateway dispatch, client APIs (sync/async REST + websocket), tg-describe-image CLI, IAM capability, and specs. Closes #879
2026-07-15 00:02:11 +02:00 · 2026-07-12 05:47:04 -06:00 · 2026-07-12 05:47:04 -06:00 · 40f01c123b
commit 40f01c123b
parent 9136526863
42 changed files with 1845 additions and 14 deletions
--- a/specs/api/components/schemas/image-to-text/ImageToTextRequest.yaml
+++ b/specs/api/components/schemas/image-to-text/ImageToTextRequest.yaml
@ -0,0 +1,27 @@
+type: object
+description: |
+  Image-to-text request - describe an image using a vision model.
+
+  The image payload is base64-encoded; raw binary is not accepted.
+required:
+  - image
+  - mime_type
+properties:
+  image:
+    type: string
+    description: Base64-encoded image payload
+    example: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
+  mime_type:
+    type: string
+    description: MIME type of the image
+    example: image/png
+  prompt:
+    type: string
+    description: |
+      Instruction for the vision model. Optional; the service applies a
+      default prompt ("Describe this image") when omitted or empty.
+    example: List the objects visible in this image.
+  system:
+    type: string
+    description: Optional system prompt that sets behavior and context for the vision model
+    example: You are an expert at describing photographs for visually impaired users.
--- a/specs/api/components/schemas/image-to-text/ImageToTextResponse.yaml
+++ b/specs/api/components/schemas/image-to-text/ImageToTextResponse.yaml
@ -0,0 +1,21 @@
+type: object
+description: Image-to-text response
+required:
+  - description
+properties:
+  description:
+    type: string
+    description: Generated description of the image
+    example: A red square on a white background.
+  in_token:
+    type: integer
+    description: Number of input tokens consumed
+    example: 245
+  out_token:
+    type: integer
+    description: Number of output tokens generated
+    example: 32
+  model:
+    type: string
+    description: Model used to describe the image
+    example: gpt-5-mini
--- a/specs/api/openapi.yaml
+++ b/specs/api/openapi.yaml
@ -52,7 +52,7 @@ info:
    Workspace context comes from the authenticated token.

    Accessed via `/api/v1/flow/{flow}/service/{kind}`:
-    - AI services: agent, text-completion, prompt, RAG (document/graph)
+    - AI services: agent, text-completion, image-to-text, prompt, RAG (document/graph)
    - Embeddings: embeddings, graph-embeddings, document-embeddings
    - Query: triples, rows, nlp-query, structured-query, sparql-query, row-embeddings
    - Data loading: text-load, document-load
@ -136,6 +136,8 @@ paths:
    $ref: './paths/flow/graph-rag.yaml'
  /api/v1/flow/{flow}/service/text-completion:
    $ref: './paths/flow/text-completion.yaml'
+  /api/v1/flow/{flow}/service/image-to-text:
+    $ref: './paths/flow/image-to-text.yaml'
  /api/v1/flow/{flow}/service/prompt:
    $ref: './paths/flow/prompt.yaml'
  /api/v1/flow/{flow}/service/embeddings:
--- a/specs/api/paths/flow/image-to-text.yaml
+++ b/specs/api/paths/flow/image-to-text.yaml
@ -0,0 +1,87 @@
+post:
+  tags:
+    - Flow Services
+  summary: Image to text - describe images with a vision model
+  description: |
+    Describe an image using a vision-capable LLM.
+
+    This is a **flow-scoped** service. It requires a flow instance
+    and operates within the workspace associated with the
+    authenticated bearer token.
+
+    ## Image-to-Text Overview
+
+    Converts image content into a text description:
+    - General image description ("what is in this picture?")
+    - Targeted extraction via a custom prompt (objects, text, layout)
+    - Behavior shaping via an optional system prompt
+
+    ## Request Fields
+
+    - **image**: Base64-encoded image payload (raw binary is not accepted)
+    - **mime_type**: Image MIME type, e.g. `image/png`, `image/jpeg`
+    - **prompt**: Optional instruction; defaults to "Describe this image"
+    - **system**: Optional system prompt for behavior and constraints
+
+    ## Token Counting
+
+    Response includes token usage:
+    - `in_token`: Input tokens (prompt + image)
+    - `out_token`: Generated tokens
+    - Useful for cost tracking and optimization
+
+    ## Availability
+
+    Image-to-text is an optional service: it is only available in flows
+    whose blueprint defines the `image-to-text` interface. Invoking it
+    on a flow without the interface returns an error.
+
+  operationId: imageToTextService
+  security:
+    - bearerAuth: []
+  parameters:
+    - name: flow
+      in: path
+      required: true
+      schema:
+        type: string
+      description: Flow instance ID
+      example: my-flow
+  requestBody:
+    required: true
+    content:
+      application/json:
+        schema:
+          $ref: '../../components/schemas/image-to-text/ImageToTextRequest.yaml'
+        examples:
+          basicDescription:
+            summary: Basic image description with default prompt
+            value:
+              image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
+              mime_type: image/png
+          targetedExtraction:
+            summary: Targeted extraction with custom prompts
+            value:
+              image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
+              mime_type: image/png
+              prompt: List all text visible in this image.
+              system: You are an OCR assistant. Reply with the extracted text only.
+  responses:
+    '200':
+      description: Successful response
+      content:
+        application/json:
+          schema:
+            $ref: '../../components/schemas/image-to-text/ImageToTextResponse.yaml'
+          examples:
+            imageDescription:
+              summary: Image description with token usage
+              value:
+                description: A single blue pixel on a transparent background.
+                in_token: 245
+                out_token: 32
+                model: gpt-5-mini
+    '401':
+      $ref: '../../components/responses/Unauthorized.yaml'
+    '500':
+      $ref: '../../components/responses/Error.yaml'
--- a/specs/websocket/asyncapi.yaml
+++ b/specs/websocket/asyncapi.yaml
@ -43,7 +43,7 @@ info:
    - config, flow, librarian, knowledge, collection-management

    **Flow-Scoped Services** (require `flow` parameter, workspace from token):
-    - agent, text-completion, prompt, document-rag, graph-rag
+    - agent, text-completion, image-to-text, prompt, document-rag, graph-rag
    - embeddings, graph-embeddings, document-embeddings
    - triples, rows, nlp-query, structured-query, sparql-query, structured-diag, row-embeddings
    - text-load, document-load, mcp-tool
--- a/specs/websocket/components/messages/ServiceRequest.yaml
+++ b/specs/websocket/components/messages/ServiceRequest.yaml
@ -24,6 +24,7 @@ payload:
    - $ref: './requests/DocumentRagRequest.yaml'
    - $ref: './requests/GraphRagRequest.yaml'
    - $ref: './requests/TextCompletionRequest.yaml'
+    - $ref: './requests/ImageToTextRequest.yaml'
    - $ref: './requests/PromptRequest.yaml'
    - $ref: './requests/EmbeddingsRequest.yaml'
    - $ref: './requests/McpToolRequest.yaml'
--- a/specs/websocket/components/messages/requests/ImageToTextRequest.yaml
+++ b/specs/websocket/components/messages/requests/ImageToTextRequest.yaml
@ -0,0 +1,28 @@
+type: object
+description: WebSocket request for image-to-text service (flow-scoped service)
+required:
+  - id
+  - service
+  - flow
+  - request
+properties:
+  id:
+    type: string
+    description: Unique request identifier
+  service:
+    type: string
+    const: image-to-text
+    description: Service identifier for image-to-text service
+  flow:
+    type: string
+    description: Flow ID
+  request:
+    $ref: '../../../../api/components/schemas/image-to-text/ImageToTextRequest.yaml'
+examples:
+  - id: req-1
+    service: image-to-text
+    flow: my-flow
+    request:
+      image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
+      mime_type: image/png
+      prompt: Describe this image
--- a/specs/websocket/components/schemas/RequestEnvelope.yaml
+++ b/specs/websocket/components/schemas/RequestEnvelope.yaml
@ -25,9 +25,10 @@ properties:

      Global services: iam
      Workspace-scoped services: config, flow, librarian, knowledge, collection-management
-      Flow-scoped services: agent, text-completion, prompt, document-rag, graph-rag,
-      embeddings, graph-embeddings, document-embeddings, triples, objects,
-      nlp-query, structured-query, structured-diag, text-load, document-load, mcp-tool
+      Flow-scoped services: agent, text-completion, image-to-text, prompt,
+      document-rag, graph-rag, embeddings, graph-embeddings, document-embeddings,
+      triples, objects, nlp-query, structured-query, structured-diag, text-load,
+      document-load, mcp-tool
    examples:
      - config
      - agent