feat: pluggable image-to-text service with OpenAI vision backend (#1038)

Adds a full-stack image description service: schema, base class,                                                                  
OpenAI backend, gateway dispatch, client APIs (sync/async REST +
websocket), tg-describe-image CLI, IAM capability, and specs.                                                                     
                                                                                                                                    
Closes #879

trustgraph-flow/pyproject.toml

@@ -85,6 +85,7 @@ graph-embeddings-write-pinecone = "trustgraph.storage.graph_embeddings.pinecone:
 graph-embeddings-write-qdrant = "trustgraph.storage.graph_embeddings.qdrant:run"
 graph-embeddings = "trustgraph.embeddings.graph_embeddings:run"
 graph-rag = "trustgraph.retrieval.graph_rag:run"
+image-to-text-openai = "trustgraph.model.image_to_text.openai:run"
 reranker-flashrank = "trustgraph.reranker.flashrank:run"
 kg-extract-agent = "trustgraph.extract.kg.agent:run"
 kg-extract-definitions = "trustgraph.extract.kg.definitions:run"

trustgraph-flow/trustgraph/gateway/dispatch/image_to_text.py

@@ -0,0 +1,32 @@
+
+from ... schema import ImageToTextRequest, ImageToTextResponse
+from ... messaging import TranslatorRegistry
+
+from . requestor import ServiceRequestor
+
+class ImageToTextRequestor(ServiceRequestor):
+    def __init__(
+            self, backend, request_queue, response_queue, timeout,
+            consumer, subscriber,
+    ):
+
+        super(ImageToTextRequestor, self).__init__(
+            backend=backend,
+            request_queue=request_queue,
+            response_queue=response_queue,
+            request_schema=ImageToTextRequest,
+            response_schema=ImageToTextResponse,
+            subscription = subscriber,
+            consumer_name = consumer,
+            timeout=timeout,
+        )
+
+        self.request_translator = TranslatorRegistry.get_request_translator("image-to-text")
+        self.response_translator = TranslatorRegistry.get_response_translator("image-to-text")
+
+    def to_request(self, body):
+        return self.request_translator.decode(body)
+
+    def from_response(self, message):
+        return self.response_translator.encode_with_completion(message)
+

trustgraph-flow/trustgraph/gateway/dispatch/manager.py

@@ -23,6 +23,7 @@ from . collection_management import CollectionManagementRequestor
 from . embeddings import EmbeddingsRequestor
 from . agent import AgentRequestor
 from . text_completion import TextCompletionRequestor
+from . image_to_text import ImageToTextRequestor
 from . prompt import PromptRequestor
 from . graph_rag import GraphRagRequestor
 from . document_rag import DocumentRagRequestor
@@ -76,6 +77,7 @@ request_response_dispatchers = {
     "row-embeddings": RowEmbeddingsQueryRequestor,
     "sparql": SparqlQueryRequestor,
     "reranker": RerankerRequestor,
+    "image-to-text": ImageToTextRequestor,
 }
 
 system_dispatchers = {

trustgraph-flow/trustgraph/gateway/registry.py

@@ -519,6 +519,7 @@ _FLOW_SERVICES = {
     "row-embeddings": "row-embeddings:read",
     "sparql": "sparql:read",
     "reranker": "reranker",
+    "image-to-text": "image-to-text",
 }
 for _kind, _cap in _FLOW_SERVICES.items():
     _register_flow_kind("flow-service", _kind, _cap)

trustgraph-flow/trustgraph/iam/service/iam.py

@@ -73,6 +73,7 @@ _READER_CAPS = {
     "llm",
     "embeddings",
     "reranker",
+    "image-to-text",
     "mcp",
     "config:read",
     "flows:read",

trustgraph-flow/trustgraph/model/image_to_text/openai/__init__.py

@@ -0,0 +1 @@
+from . service import *

trustgraph-flow/trustgraph/model/image_to_text/openai/__main__.py

@@ -0,0 +1,6 @@
+#!/usr/bin/env python3
+
+from . service import run
+
+if __name__ == '__main__':
+    run()

trustgraph-flow/trustgraph/model/image_to_text/openai/service.py

@@ -0,0 +1,172 @@
+
+"""
+Simple image-to-text service, describes images using the OpenAI vision
+API.  Input is base64-encoded image, MIME type and prompt, output is
+image description.
+"""
+
+from openai import OpenAI, RateLimitError, InternalServerError
+import os
+import logging
+
+from .... exceptions import TooManyRequests, LlmError
+from .... base import ImageToTextService, ImageDescriptionResult
+
+# Module logger
+logger = logging.getLogger(__name__)
+
+default_ident = "image-to-text"
+
+default_model = 'gpt-5-mini'
+default_max_output = 4096
+default_api_key = os.getenv("OPENAI_TOKEN")
+default_base_url = os.getenv("OPENAI_BASE_URL")
+default_prompt = 'Describe this image'
+
+if default_base_url is None or default_base_url == "":
+    default_base_url = "https://api.openai.com/v1"
+
+class Processor(ImageToTextService):
+
+    def __init__(self, **params):
+
+        model = params.get("model", default_model)
+        api_key = params.get("api_key", default_api_key)
+        base_url = params.get("url", default_base_url)
+        max_output = params.get("max_output", default_max_output)
+
+        if not api_key:
+            api_key = "not-set"
+
+        super(Processor, self).__init__(
+            **params | {
+                "model": model,
+                "max_output": max_output,
+                "base_url": base_url,
+            }
+        )
+
+        self.default_model = model
+        self.max_output = max_output
+
+        if base_url:
+            self.openai = OpenAI(base_url=base_url, api_key=api_key)
+        else:
+            self.openai = OpenAI(api_key=api_key)
+
+        logger.info("OpenAI image-to-text service initialized")
+
+    async def describe_image(
+        self, image, mime_type, prompt, system, model=None,
+    ):
+
+        model_name = model or self.default_model
+
+        logger.debug(f"Using model: {model_name}")
+
+        if not prompt:
+            prompt = default_prompt
+
+        if system:
+            prompt = system + "\n\n" + prompt
+
+        try:
+
+            messages = [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": prompt
+                        },
+                        {
+                            "type": "image_url",
+                            "image_url": {
+                                "url": f"data:{mime_type};base64,{image}"
+                            }
+                        }
+                    ]
+                }
+            ]
+
+            resp = self.openai.chat.completions.create(
+                model=model_name,
+                messages=messages,
+                max_completion_tokens=self.max_output,
+            )
+
+            inputtokens = resp.usage.prompt_tokens
+            outputtokens = resp.usage.completion_tokens
+
+            content = resp.choices[0].message.content
+
+            logger.debug(f"Image description: {content}")
+            logger.info(f"Input Tokens: {inputtokens}")
+            logger.info(f"Output Tokens: {outputtokens}")
+
+            resp = ImageDescriptionResult(
+                text = content,
+                in_token = inputtokens,
+                out_token = outputtokens,
+                model = model_name
+            )
+
+            return resp
+
+        except RateLimitError as e:
+            try:
+                body = getattr(e, 'body', {})
+                if isinstance(body, dict):
+                    code = body.get('error', {}).get('code')
+                    if code in ('insufficient_quota', 'invalid_api_key', 'account_deactivated'):
+                        raise RuntimeError(f"OpenAI unrecoverable error: {code} - {body['error'].get('message', '')}")
+            except (ValueError, KeyError, TypeError, AttributeError):
+                pass
+            # Leave rate limit retries to the base handler
+            raise TooManyRequests()
+
+        except InternalServerError:
+            # Treat 503 as a retryable LlmError
+            raise LlmError()
+
+        except Exception as e:
+
+            # Apart from rate limits, treat all exceptions as unrecoverable
+
+            logger.error(f"OpenAI image-to-text exception ({type(e).__name__}): {e}", exc_info=True)
+            raise e
+
+    @staticmethod
+    def add_args(parser):
+
+        ImageToTextService.add_args(parser)
+
+        parser.add_argument(
+            '-m', '--model',
+            default=default_model,
+            help=f'Vision model (default: {default_model})'
+        )
+
+        parser.add_argument(
+            '-k', '--api-key',
+            default=default_api_key,
+            help=f'OpenAI API key'
+        )
+
+        parser.add_argument(
+            '-u', '--url',
+            default=default_base_url,
+            help=f'OpenAI service base URL'
+        )
+
+        parser.add_argument(
+            '-x', '--max-output',
+            type=int,
+            default=default_max_output,
+            help=f'Vision model max output tokens (default: {default_max_output})'
+        )
+
+def run():
+
+    Processor.launch(default_ident, __doc__)