apunkt/trustgraph
1
0
Fork 0
mirror of https://github.com/trustgraph-ai/trustgraph.git synced 2026-07-15 00:02:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

feat: pluggable image-to-text service with OpenAI vision backend (#1038)

Browse source 
Adds a full-stack image description service: schema, base class,                                                                  
OpenAI backend, gateway dispatch, client APIs (sync/async REST +
websocket), tg-describe-image CLI, IAM capability, and specs.                                                                     
                                                                                                                                    
Closes #879
This commit is contained in:
Sunny Yang 2026-07-12 05:47:04 -06:00 • committed by GitHub
parent 9136526863
commit 40f01c123b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
42 changed files with 1845 additions and 14 deletions
Download patch file Download diff file Expand all files Collapse all files

2
trustgraph-base/trustgraph/api/__init__.py
View file

@ -107,6 +107,7 @@ from .types import (
AgentAnswer,
RAGChunk,
TextCompletionResult,
ImageToTextResult,
ProvenanceEvent,
)
@ -186,6 +187,7 @@ __all__ = [
"AgentAnswer",
"RAGChunk",
"TextCompletionResult",
"ImageToTextResult",
"ProvenanceEvent",
# Exceptions

53
trustgraph-base/trustgraph/api/async_flow.py
View file

@ -12,9 +12,10 @@ AsyncSocketClient instead.
import aiohttp
import json
import base64
from typing import Optional, Dict, Any, List
from . types import TextCompletionResult
from . types import TextCompletionResult, ImageToTextResult
from . exceptions import ProtocolException, ApplicationException
@ -476,6 +477,56 @@ class AsyncFlowInstance:
model=result.get("model"),
)
async def image_to_text(self, image: bytes, mime_type: str,
prompt: Optional[str] = None,
system: Optional[str] = None,
**kwargs: Any) -> ImageToTextResult:
"""
Describe an image using the image-to-text service (non-streaming).
Args:
image: Image content as bytes
mime_type: Image MIME type (e.g. "image/jpeg")
prompt: Optional user prompt (backend default used if None)
system: Optional system prompt
**kwargs: Additional service-specific parameters
Returns:
ImageToTextResult: Result with text, in_token, out_token, model
Example:
```python
async_flow = await api.async_flow()
flow = async_flow.id("default")
with open("photo.jpg", "rb") as f:
result = await flow.image_to_text(
image=f.read(),
mime_type="image/jpeg",
)
print(result.text)
print(f"Tokens: {result.in_token} in, {result.out_token} out")
```
"""
# The image rides the JSON wire format as base64 text
request_data = {
"image": base64.b64encode(image).decode("utf-8"),
"mime_type": mime_type,
}
if prompt is not None:
request_data["prompt"] = prompt
if system is not None:
request_data["system"] = system
request_data.update(kwargs)
result = await self.request("image-to-text", request_data)
return ImageToTextResult(
text=result.get("description", ""),
in_token=result.get("in_token"),
out_token=result.get("out_token"),
model=result.get("model"),
)
async def graph_rag(self, query: str, collection: str,
max_subgraph_size: int = 1000, max_subgraph_count: int = 5,
max_entity_distance: int = 3, **kwargs: Any) -> str:

37
trustgraph-base/trustgraph/api/async_socket_client.py
View file

@ -1,11 +1,12 @@
import json
import base64
import asyncio
import websockets
from typing import Optional, Dict, Any, AsyncIterator, Union
from . types import AgentThought, AgentObservation, AgentAnswer, RAGChunk, TextCompletionResult
from . exceptions import ProtocolException, ApplicationException
from . types import AgentThought, AgentObservation, AgentAnswer, RAGChunk, TextCompletionResult, ImageToTextResult
from . exceptions import ProtocolException, ApplicationException, raise_from_error_dict
class AsyncSocketClient:
@ -353,6 +354,38 @@ class AsyncSocketFlowInstance:
if isinstance(chunk, RAGChunk):
yield chunk
async def image_to_text(self, image: bytes, mime_type: str,
prompt: Optional[str] = None,
system: Optional[str] = None,
**kwargs) -> ImageToTextResult:
"""Describe an image using the image-to-text service (non-streaming).
Returns an ImageToTextResult with the description text and token counts.
"""
# The image rides the JSON wire format as base64 text
request = {
"image": base64.b64encode(image).decode("utf-8"),
"mime_type": mime_type,
}
if prompt is not None:
request["prompt"] = prompt
if system is not None:
request["system"] = system
request.update(kwargs)
result = await self.client._send_request("image-to-text", self.flow_id, request)
# Service errors arrive inside the response body
if isinstance(result, dict) and result.get("error"):
raise_from_error_dict(result["error"])
return ImageToTextResult(
text=result.get("description", ""),
in_token=result.get("in_token"),
out_token=result.get("out_token"),
model=result.get("model"),
)
async def graph_rag(self, query: str, collection: str,
max_subgraph_size: int = 1000, max_subgraph_count: int = 5,
max_entity_distance: int = 3, streaming: bool = False, **kwargs):

50
trustgraph-base/trustgraph/api/flow.py
View file

@ -11,7 +11,7 @@ import base64
from .. knowledge import hash, Uri, Literal, QuotedTriple
from .. schema import IRI, LITERAL, TRIPLE
from . types import Triple, TextCompletionResult
from . types import Triple, TextCompletionResult, ImageToTextResult
from . exceptions import ProtocolException
@ -296,6 +296,54 @@ class FlowInstance:
model=result.get("model"),
)
def image_to_text(self, image, mime_type, prompt=None, system=None):
"""
Describe an image using the flow's image-to-text service.
Args:
image: Image content as bytes
mime_type: Image MIME type (e.g. "image/jpeg")
prompt: Optional user prompt (backend default used if None)
system: Optional system prompt
Returns:
ImageToTextResult: Result with text, in_token, out_token, model
Example:
```python
flow = api.flow().id("default")
with open("photo.jpg", "rb") as f:
result = flow.image_to_text(
image=f.read(),
mime_type="image/jpeg",
)
print(result.text)
print(f"Tokens: {result.in_token} in, {result.out_token} out")
```
"""
# The image rides the JSON wire format as base64 text
input = {
"image": base64.b64encode(image).decode("utf-8"),
"mime_type": mime_type,
}
if prompt is not None:
input["prompt"] = prompt
if system is not None:
input["system"] = system
result = self.request(
"service/image-to-text",
input
)
return ImageToTextResult(
text=result.get("description", ""),
in_token=result.get("in_token"),
out_token=result.get("out_token"),
model=result.get("model"),
)
def agent(self, question,state=None, group=None, history=None):
"""
Execute an agent operation with reasoning and tool use capabilities.

35
trustgraph-base/trustgraph/api/socket_client.py
View file

@ -9,13 +9,14 @@ multiplexes requests by ID.
"""
import json
import base64
import asyncio
import websockets
from websockets.exceptions import ConnectionClosed
from typing import Optional, Dict, Any, Iterator, Union, List
from threading import Lock
from . types import AgentThought, AgentObservation, AgentAnswer, RAGChunk, StreamingChunk, ProvenanceEvent, TextCompletionResult
from . types import AgentThought, AgentObservation, AgentAnswer, RAGChunk, StreamingChunk, ProvenanceEvent, TextCompletionResult, ImageToTextResult
from . exceptions import ProtocolException, raise_from_error_dict
@ -673,6 +674,38 @@ class SocketFlowInstance:
if isinstance(chunk, RAGChunk):
yield chunk
def image_to_text(self, image: bytes, mime_type: str,
prompt: Optional[str] = None,
system: Optional[str] = None,
**kwargs: Any) -> ImageToTextResult:
"""Describe an image using the image-to-text service (non-streaming).
Returns an ImageToTextResult with the description text and token counts.
"""
# The image rides the JSON wire format as base64 text
request = {
"image": base64.b64encode(image).decode("utf-8"),
"mime_type": mime_type,
}
if prompt is not None:
request["prompt"] = prompt
if system is not None:
request["system"] = system
request.update(kwargs)
result = self.client._send_request_sync("image-to-text", self.flow_id, request, False)
# Service errors arrive inside the response body
if isinstance(result, dict) and result.get("error"):
raise_from_error_dict(result["error"])
return ImageToTextResult(
text=result.get("description", ""),
in_token=result.get("in_token"),
out_token=result.get("out_token"),
model=result.get("model"),
)
def graph_rag(
self,
query: str,

18
trustgraph-base/trustgraph/api/types.py
View file

@ -240,6 +240,24 @@ class TextCompletionResult:
model: Optional[str] = None
sources: List[Dict[str, str]] = dataclasses.field(default_factory=list)
@dataclasses.dataclass
class ImageToTextResult:
"""
Result from an image-to-text request.
Returned by image_to_text(). Non-streaming only.
Attributes:
text: Text description of the image
in_token: Input token count (None if not available)
out_token: Output token count (None if not available)
model: Model identifier (None if not available)
"""
text: Optional[str]
in_token: Optional[int] = None
out_token: Optional[int] = None
model: Optional[str] = None
@dataclasses.dataclass
class ProvenanceEvent:
"""

1
trustgraph-base/trustgraph/base/__init__.py
View file

@ -44,6 +44,7 @@ from . agent_client import AgentClientSpec
from . structured_query_client import StructuredQueryClientSpec
from . reranker_client import RerankerClientSpec
from . reranker_service import RerankerService
from . image_to_text_service import ImageToTextService, ImageDescriptionResult
from . keyword_index_service import KeywordIndexService
from . keyword_index_client import KeywordIndexClientSpec, KeywordIndexClient
from . row_embeddings_query_client import RowEmbeddingsQueryClientSpec

170
trustgraph-base/trustgraph/base/image_to_text_service.py Normal file
View file

@ -0,0 +1,170 @@
"""
Image-to-text description base class
"""
from __future__ import annotations
from argparse import ArgumentParser
import logging
from prometheus_client import Histogram, Info
from .. schema import ImageToTextRequest, ImageToTextResponse, Error
from .. exceptions import TooManyRequests
from .. base import FlowProcessor, ConsumerSpec, ProducerSpec, ParameterSpec
# Module logger
logger = logging.getLogger(__name__)
default_ident = "image-to-text"
default_concurrency = 1
class ImageDescriptionResult:
def __init__(
self, text = None, in_token = None, out_token = None,
model = None,
):
self.text = text
self.in_token = in_token
self.out_token = out_token
self.model = model
__slots__ = ["text", "in_token", "out_token", "model"]
class ImageToTextService(FlowProcessor):
"""
Extensible service processing image description requests.
This class handles the core logic of dispatching image-to-text
requests to integrated underlying vision model providers
(e.g. OpenAI).
"""
def __init__(self, **params):
id = params.get("id", default_ident)
concurrency = params.get("concurrency", 1)
super(ImageToTextService, self).__init__(**params | {
"id": id,
"concurrency": concurrency,
})
self.register_specification(
ConsumerSpec(
name = "request",
schema = ImageToTextRequest,
handler = self.on_request,
concurrency = concurrency,
)
)
self.register_specification(
ProducerSpec(
name = "response",
schema = ImageToTextResponse
)
)
self.register_specification(
ParameterSpec(
name = "model",
)
)
if not hasattr(__class__, "image_to_text_metric"):
__class__.image_to_text_metric = Histogram(
'image_to_text_duration',
'Image-to-text duration (seconds)',
["id", "flow"],
buckets=[
0.25, 0.5, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0,
8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, 16.0,
17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0,
30.0, 35.0, 40.0, 45.0, 50.0, 60.0, 80.0, 100.0,
120.0
]
)
if not hasattr(__class__, "image_to_text_model_metric"):
__class__.image_to_text_model_metric = Info(
'image_to_text_model',
'Image-to-text model',
["processor", "flow"]
)
async def on_request(self, msg, consumer, flow):
try:
request = msg.value()
# Sender-produced ID
id = msg.properties()["id"]
model = flow("model")
with __class__.image_to_text_metric.labels(
id=self.id,
flow=f"{flow.name}-{consumer.name}",
).time():
response = await self.describe_image(
request.image, request.mime_type,
request.prompt, request.system, model,
)
await flow("response").send(
ImageToTextResponse(
error=None,
description=response.text,
in_token=response.in_token,
out_token=response.out_token,
model=response.model,
),
properties={"id": id}
)
__class__.image_to_text_model_metric.labels(
processor = self.id,
flow = flow.name
).info({
"model": str(model) if model is not None else "",
})
except TooManyRequests as e:
raise e
except Exception as e:
# Apart from rate limits, treat all exceptions as unrecoverable
logger.error(f"Image-to-text service exception: {e}", exc_info=True)
logger.debug("Sending error response...")
await flow.producer["response"].send(
ImageToTextResponse(
error=Error(
type = "image-to-text-error",
message = str(e),
),
description=None,
in_token=None,
out_token=None,
model=None,
),
properties={"id": id}
)
@staticmethod
def add_args(parser: ArgumentParser) -> None:
parser.add_argument(
'-c', '--concurrency',
type=int,
default=default_concurrency,
help=f'Concurrent processing threads (default: {default_concurrency})'
)
FlowProcessor.add_args(parser)

7
trustgraph-base/trustgraph/messaging/__init__.py
View file

@ -5,6 +5,7 @@ from .translators import *
from .translators.agent import AgentRequestTranslator, AgentResponseTranslator
from .translators.embeddings import EmbeddingsRequestTranslator, EmbeddingsResponseTranslator
from .translators.text_completion import TextCompletionRequestTranslator, TextCompletionResponseTranslator
from .translators.image_to_text import ImageToTextRequestTranslator, ImageToTextResponseTranslator
from .translators.retrieval import (
DocumentRagRequestTranslator, DocumentRagResponseTranslator,
GraphRagRequestTranslator, GraphRagResponseTranslator
@ -50,6 +51,12 @@ TranslatorRegistry.register_service(
TextCompletionResponseTranslator()
)
TranslatorRegistry.register_service(
"image-to-text",
ImageToTextRequestTranslator(),
ImageToTextResponseTranslator()
)
TranslatorRegistry.register_service(
"document-rag",
DocumentRagRequestTranslator(),

52
trustgraph-base/trustgraph/messaging/translators/image_to_text.py Normal file
View file

@ -0,0 +1,52 @@
import base64
from typing import Dict, Any, Tuple
from ...schema import ImageToTextRequest, ImageToTextResponse
from .base import MessageTranslator
class ImageToTextRequestTranslator(MessageTranslator):
"""Translator for ImageToTextRequest schema objects"""
def decode(self, data: Dict[str, Any]) -> ImageToTextRequest:
# Base64 content validation only. The image field carries
# base64 text end-to-end: raw binary can't ride the JSON wire
# format, and the payload passes through unchanged
base64.b64decode(data["image"], validate=True)
return ImageToTextRequest(
image=data["image"],
mime_type=data["mime_type"],
prompt=data.get("prompt", ""),
system=data.get("system", ""),
)
def encode(self, obj: ImageToTextRequest) -> Dict[str, Any]:
return {
"image": obj.image,
"mime_type": obj.mime_type,
"prompt": obj.prompt,
"system": obj.system,
}
class ImageToTextResponseTranslator(MessageTranslator):
"""Translator for ImageToTextResponse schema objects"""
def decode(self, data: Dict[str, Any]) -> ImageToTextResponse:
raise NotImplementedError("Response translation to Pulsar not typically needed")
def encode(self, obj: ImageToTextResponse) -> Dict[str, Any]:
result = {"description": obj.description}
if obj.in_token is not None:
result["in_token"] = obj.in_token
if obj.out_token is not None:
result["out_token"] = obj.out_token
if obj.model is not None:
result["model"] = obj.model
return result
def encode_with_completion(self, obj: ImageToTextResponse) -> Tuple[Dict[str, Any], bool]:
"""Returns (response_dict, is_final). Image-to-text is non-streaming."""
return self.encode(obj), True

3
trustgraph-base/trustgraph/schema/services/__init__.py
View file

@ -17,4 +17,5 @@ from .storage import *
from .tool_service import *
from .sparql_query import *
from .reranker import *
from .audit import *
from .audit import *
from .image_to_text import *

24
trustgraph-base/trustgraph/schema/services/image_to_text.py Normal file
View file

@ -0,0 +1,24 @@
from dataclasses import dataclass
from ..core.primitives import Error
############################################################################
# Image to text
@dataclass
class ImageToTextRequest:
# Image payload: base64-encoded image data
image: str = ""
mime_type: str = ""
prompt: str = ""
system: str = ""
@dataclass
class ImageToTextResponse:
error: Error | None = None
description: str = ""
in_token: int | None = None
out_token: int | None = None
model: str | None = None