mirror of
https://github.com/trustgraph-ai/trustgraph.git
synced 2026-07-14 15:52:11 +02:00
feat: pluggable image-to-text service with OpenAI vision backend (#1038)
Adds a full-stack image description service: schema, base class,
OpenAI backend, gateway dispatch, client APIs (sync/async REST +
websocket), tg-describe-image CLI, IAM capability, and specs.
Closes #879
This commit is contained in:
parent
9136526863
commit
40f01c123b
42 changed files with 1845 additions and 14 deletions
|
|
@ -0,0 +1,27 @@
|
|||
type: object
|
||||
description: |
|
||||
Image-to-text request - describe an image using a vision model.
|
||||
|
||||
The image payload is base64-encoded; raw binary is not accepted.
|
||||
required:
|
||||
- image
|
||||
- mime_type
|
||||
properties:
|
||||
image:
|
||||
type: string
|
||||
description: Base64-encoded image payload
|
||||
example: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
|
||||
mime_type:
|
||||
type: string
|
||||
description: MIME type of the image
|
||||
example: image/png
|
||||
prompt:
|
||||
type: string
|
||||
description: |
|
||||
Instruction for the vision model. Optional; the service applies a
|
||||
default prompt ("Describe this image") when omitted or empty.
|
||||
example: List the objects visible in this image.
|
||||
system:
|
||||
type: string
|
||||
description: Optional system prompt that sets behavior and context for the vision model
|
||||
example: You are an expert at describing photographs for visually impaired users.
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
type: object
|
||||
description: Image-to-text response
|
||||
required:
|
||||
- description
|
||||
properties:
|
||||
description:
|
||||
type: string
|
||||
description: Generated description of the image
|
||||
example: A red square on a white background.
|
||||
in_token:
|
||||
type: integer
|
||||
description: Number of input tokens consumed
|
||||
example: 245
|
||||
out_token:
|
||||
type: integer
|
||||
description: Number of output tokens generated
|
||||
example: 32
|
||||
model:
|
||||
type: string
|
||||
description: Model used to describe the image
|
||||
example: gpt-5-mini
|
||||
|
|
@ -52,7 +52,7 @@ info:
|
|||
Workspace context comes from the authenticated token.
|
||||
|
||||
Accessed via `/api/v1/flow/{flow}/service/{kind}`:
|
||||
- AI services: agent, text-completion, prompt, RAG (document/graph)
|
||||
- AI services: agent, text-completion, image-to-text, prompt, RAG (document/graph)
|
||||
- Embeddings: embeddings, graph-embeddings, document-embeddings
|
||||
- Query: triples, rows, nlp-query, structured-query, sparql-query, row-embeddings
|
||||
- Data loading: text-load, document-load
|
||||
|
|
@ -136,6 +136,8 @@ paths:
|
|||
$ref: './paths/flow/graph-rag.yaml'
|
||||
/api/v1/flow/{flow}/service/text-completion:
|
||||
$ref: './paths/flow/text-completion.yaml'
|
||||
/api/v1/flow/{flow}/service/image-to-text:
|
||||
$ref: './paths/flow/image-to-text.yaml'
|
||||
/api/v1/flow/{flow}/service/prompt:
|
||||
$ref: './paths/flow/prompt.yaml'
|
||||
/api/v1/flow/{flow}/service/embeddings:
|
||||
|
|
|
|||
87
specs/api/paths/flow/image-to-text.yaml
Normal file
87
specs/api/paths/flow/image-to-text.yaml
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
post:
|
||||
tags:
|
||||
- Flow Services
|
||||
summary: Image to text - describe images with a vision model
|
||||
description: |
|
||||
Describe an image using a vision-capable LLM.
|
||||
|
||||
This is a **flow-scoped** service. It requires a flow instance
|
||||
and operates within the workspace associated with the
|
||||
authenticated bearer token.
|
||||
|
||||
## Image-to-Text Overview
|
||||
|
||||
Converts image content into a text description:
|
||||
- General image description ("what is in this picture?")
|
||||
- Targeted extraction via a custom prompt (objects, text, layout)
|
||||
- Behavior shaping via an optional system prompt
|
||||
|
||||
## Request Fields
|
||||
|
||||
- **image**: Base64-encoded image payload (raw binary is not accepted)
|
||||
- **mime_type**: Image MIME type, e.g. `image/png`, `image/jpeg`
|
||||
- **prompt**: Optional instruction; defaults to "Describe this image"
|
||||
- **system**: Optional system prompt for behavior and constraints
|
||||
|
||||
## Token Counting
|
||||
|
||||
Response includes token usage:
|
||||
- `in_token`: Input tokens (prompt + image)
|
||||
- `out_token`: Generated tokens
|
||||
- Useful for cost tracking and optimization
|
||||
|
||||
## Availability
|
||||
|
||||
Image-to-text is an optional service: it is only available in flows
|
||||
whose blueprint defines the `image-to-text` interface. Invoking it
|
||||
on a flow without the interface returns an error.
|
||||
|
||||
operationId: imageToTextService
|
||||
security:
|
||||
- bearerAuth: []
|
||||
parameters:
|
||||
- name: flow
|
||||
in: path
|
||||
required: true
|
||||
schema:
|
||||
type: string
|
||||
description: Flow instance ID
|
||||
example: my-flow
|
||||
requestBody:
|
||||
required: true
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '../../components/schemas/image-to-text/ImageToTextRequest.yaml'
|
||||
examples:
|
||||
basicDescription:
|
||||
summary: Basic image description with default prompt
|
||||
value:
|
||||
image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
|
||||
mime_type: image/png
|
||||
targetedExtraction:
|
||||
summary: Targeted extraction with custom prompts
|
||||
value:
|
||||
image: iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==
|
||||
mime_type: image/png
|
||||
prompt: List all text visible in this image.
|
||||
system: You are an OCR assistant. Reply with the extracted text only.
|
||||
responses:
|
||||
'200':
|
||||
description: Successful response
|
||||
content:
|
||||
application/json:
|
||||
schema:
|
||||
$ref: '../../components/schemas/image-to-text/ImageToTextResponse.yaml'
|
||||
examples:
|
||||
imageDescription:
|
||||
summary: Image description with token usage
|
||||
value:
|
||||
description: A single blue pixel on a transparent background.
|
||||
in_token: 245
|
||||
out_token: 32
|
||||
model: gpt-5-mini
|
||||
'401':
|
||||
$ref: '../../components/responses/Unauthorized.yaml'
|
||||
'500':
|
||||
$ref: '../../components/responses/Error.yaml'
|
||||
Loading…
Add table
Add a link
Reference in a new issue