558 lines
22 KiB
Python
558 lines
22 KiB
Python
"""Translation between the Anthropic **Messages API** and **Chat Completions**.
|
|
|
|
The router speaks Chat Completions to its local backends (Ollama, llama-server,
|
|
llama-swap). To expose ``/v1/messages`` transparently on top of that, this module
|
|
converts in both directions:
|
|
|
|
* request: Anthropic ``system`` / ``messages`` / ``tools`` → chat ``messages`` / ``tools``
|
|
* response: chat ``choices[0].message`` → Anthropic ``content`` blocks
|
|
* stream: chat completion deltas → Anthropic typed SSE events
|
|
|
|
Pure functions / a stream-translator class — no I/O, mirroring ``requests/responses.py``.
|
|
The native passthrough path (configured ``anthropic_endpoints``) does not use this
|
|
module; it forwards the Anthropic wire format straight through.
|
|
|
|
Reasoning: an inbound ``thinking`` block maps to the backend's ``reasoning_effort``;
|
|
a backend that streams ``reasoning_content`` is surfaced back as Anthropic
|
|
``thinking`` content blocks / ``thinking_delta`` events.
|
|
"""
|
|
import secrets
|
|
|
|
import orjson
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Request direction: Anthropic Messages → Chat Completions
|
|
# ---------------------------------------------------------------------------
|
|
def _system_to_text(system):
|
|
"""Flatten Anthropic ``system`` (string or text blocks) to a plain string."""
|
|
if system is None:
|
|
return None
|
|
if isinstance(system, str):
|
|
return system
|
|
if isinstance(system, list):
|
|
parts = []
|
|
for b in system:
|
|
if isinstance(b, dict) and b.get("type") == "text":
|
|
parts.append(b.get("text", ""))
|
|
elif isinstance(b, str):
|
|
parts.append(b)
|
|
return "\n\n".join(p for p in parts if p) or None
|
|
return None
|
|
|
|
|
|
def _image_block_to_chat(block):
|
|
"""Convert an Anthropic ``image`` block to an OpenAI ``image_url`` part (or None)."""
|
|
src = block.get("source") or {}
|
|
stype = src.get("type")
|
|
if stype == "base64":
|
|
media = src.get("media_type", "image/png")
|
|
data = src.get("data", "")
|
|
return {"type": "image_url", "image_url": {"url": f"data:{media};base64,{data}"}}
|
|
if stype == "url" and src.get("url"):
|
|
return {"type": "image_url", "image_url": {"url": src["url"]}}
|
|
return None
|
|
|
|
|
|
def _tool_result_content_to_str(content):
|
|
"""Flatten an Anthropic ``tool_result`` content (string or blocks) to a string."""
|
|
if content is None:
|
|
return ""
|
|
if isinstance(content, str):
|
|
return content
|
|
if isinstance(content, list):
|
|
parts = []
|
|
for b in content:
|
|
if isinstance(b, dict):
|
|
if b.get("type") == "text":
|
|
parts.append(b.get("text", ""))
|
|
elif b.get("type") == "image":
|
|
parts.append("[image]")
|
|
else:
|
|
parts.append(orjson.dumps(b).decode("utf-8"))
|
|
else:
|
|
parts.append(str(b))
|
|
return "\n".join(parts)
|
|
return orjson.dumps(content).decode("utf-8")
|
|
|
|
|
|
def _user_content_to_chat(content):
|
|
"""Split an Anthropic user ``content`` into ``(chat_content, tool_messages)``.
|
|
|
|
``tool_result`` blocks become standalone OpenAI ``role:"tool"`` messages; the
|
|
remaining text/image parts collapse to a chat ``content`` (string or list).
|
|
"""
|
|
tool_messages = []
|
|
if content is None or isinstance(content, str):
|
|
return content, tool_messages
|
|
parts = []
|
|
for b in content if isinstance(content, list) else []:
|
|
if not isinstance(b, dict):
|
|
parts.append({"type": "text", "text": str(b)})
|
|
continue
|
|
btype = b.get("type")
|
|
if btype == "text":
|
|
parts.append({"type": "text", "text": b.get("text", "")})
|
|
elif btype == "image":
|
|
img = _image_block_to_chat(b)
|
|
if img:
|
|
parts.append(img)
|
|
elif btype == "tool_result":
|
|
tool_messages.append({
|
|
"role": "tool",
|
|
"tool_call_id": b.get("tool_use_id"),
|
|
"content": _tool_result_content_to_str(b.get("content")),
|
|
})
|
|
# document / other blocks: no chat equivalent → skip
|
|
if not parts:
|
|
chat_content = None
|
|
elif len(parts) == 1 and parts[0].get("type") == "text":
|
|
chat_content = parts[0]["text"]
|
|
else:
|
|
chat_content = parts
|
|
return chat_content, tool_messages
|
|
|
|
|
|
def _assistant_content_to_chat(content):
|
|
"""Convert an Anthropic assistant ``content`` to ``(text_or_parts, tool_calls)``."""
|
|
if content is None or isinstance(content, str):
|
|
return content, None
|
|
text_parts = []
|
|
tool_calls = []
|
|
for b in content if isinstance(content, list) else []:
|
|
if not isinstance(b, dict):
|
|
continue
|
|
btype = b.get("type")
|
|
if btype == "text":
|
|
text_parts.append(b.get("text", ""))
|
|
elif btype == "tool_use":
|
|
tool_calls.append({
|
|
"id": b.get("id"),
|
|
"type": "function",
|
|
"function": {
|
|
"name": b.get("name"),
|
|
"arguments": orjson.dumps(b.get("input") or {}).decode("utf-8"),
|
|
},
|
|
})
|
|
# thinking blocks in history have no chat equivalent → drop
|
|
text = "".join(text_parts) if text_parts else None
|
|
return text, (tool_calls or None)
|
|
|
|
|
|
def anthropic_messages_to_chat(system, messages):
|
|
"""Build a Chat Completions ``messages`` list from Anthropic ``system`` + ``messages``."""
|
|
chat = []
|
|
sys_text = _system_to_text(system)
|
|
if sys_text:
|
|
chat.append({"role": "system", "content": sys_text})
|
|
for m in messages or []:
|
|
role = m.get("role")
|
|
content = m.get("content")
|
|
if role == "user":
|
|
chat_content, tool_messages = _user_content_to_chat(content)
|
|
# Anthropic packs tool results into a user turn; OpenAI wants each as a
|
|
# separate tool message *before* any trailing user text.
|
|
chat.extend(tool_messages)
|
|
if chat_content is not None and chat_content != []:
|
|
chat.append({"role": "user", "content": chat_content})
|
|
elif role == "assistant":
|
|
text, tool_calls = _assistant_content_to_chat(content)
|
|
msg = {"role": "assistant", "content": text}
|
|
if tool_calls:
|
|
msg["tool_calls"] = tool_calls
|
|
chat.append(msg)
|
|
else:
|
|
chat.append({"role": role or "user", "content": content})
|
|
return chat
|
|
|
|
|
|
def tools_anthropic_to_chat(tools):
|
|
"""Map Anthropic tool definitions → Chat Completions function tools."""
|
|
if not tools:
|
|
return None
|
|
out = []
|
|
for t in tools:
|
|
if not isinstance(t, dict):
|
|
continue
|
|
# Server tools (web_search, …) carry a ``type`` but no ``input_schema`` —
|
|
# local backends can't run them, so skip.
|
|
if "input_schema" not in t and t.get("type") not in (None, "custom"):
|
|
continue
|
|
fn = {"name": t.get("name")}
|
|
if t.get("description"):
|
|
fn["description"] = t["description"]
|
|
fn["parameters"] = t.get("input_schema") or {"type": "object", "properties": {}}
|
|
out.append({"type": "function", "function": fn})
|
|
return out or None
|
|
|
|
|
|
def tool_choice_anthropic_to_chat(tool_choice):
|
|
"""Map Anthropic ``tool_choice`` → Chat Completions ``tool_choice``."""
|
|
if not isinstance(tool_choice, dict):
|
|
return None
|
|
ttype = tool_choice.get("type")
|
|
if ttype == "auto":
|
|
return "auto"
|
|
if ttype == "any":
|
|
return "required"
|
|
if ttype == "none":
|
|
return "none"
|
|
if ttype == "tool" and tool_choice.get("name"):
|
|
return {"type": "function", "function": {"name": tool_choice["name"]}}
|
|
return None
|
|
|
|
|
|
def _thinking_to_reasoning_effort(thinking):
|
|
"""Map an Anthropic ``thinking`` config to an OpenAI ``reasoning_effort`` level."""
|
|
if not isinstance(thinking, dict):
|
|
return None
|
|
if thinking.get("type") == "disabled":
|
|
return None
|
|
budget = thinking.get("budget_tokens")
|
|
if not isinstance(budget, int):
|
|
return "medium" # adaptive / enabled without an explicit budget
|
|
if budget < 2048:
|
|
return "low"
|
|
if budget < 8192:
|
|
return "medium"
|
|
return "high"
|
|
|
|
|
|
def anthropic_to_chat_send_params(payload, chat_messages, model):
|
|
"""Assemble the Chat Completions request body from an Anthropic payload."""
|
|
send = {"messages": chat_messages, "model": model}
|
|
if payload.get("max_tokens") is not None:
|
|
send["max_tokens"] = payload["max_tokens"]
|
|
opt = {
|
|
"temperature": payload.get("temperature"),
|
|
"top_p": payload.get("top_p"),
|
|
"stop": payload.get("stop_sequences"),
|
|
"tools": tools_anthropic_to_chat(payload.get("tools")),
|
|
"tool_choice": tool_choice_anthropic_to_chat(payload.get("tool_choice")),
|
|
"reasoning_effort": _thinking_to_reasoning_effort(payload.get("thinking")),
|
|
}
|
|
send.update({k: v for k, v in opt.items() if v is not None})
|
|
return send
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Response direction: Chat Completions → Anthropic Messages
|
|
# ---------------------------------------------------------------------------
|
|
def new_message_id():
|
|
return f"msg_{secrets.token_hex(24)}"
|
|
|
|
|
|
_STOP_REASON_MAP = {
|
|
"stop": "end_turn",
|
|
"length": "max_tokens",
|
|
"tool_calls": "tool_use",
|
|
"function_call": "tool_use",
|
|
"content_filter": "end_turn",
|
|
}
|
|
|
|
|
|
def finish_reason_to_stop_reason(finish_reason, has_tool_use=False):
|
|
if has_tool_use:
|
|
return "tool_use"
|
|
return _STOP_REASON_MAP.get(finish_reason, "end_turn")
|
|
|
|
|
|
def _cached_prompt_tokens(usage):
|
|
"""Read ``prompt_tokens_details.cached_tokens`` from a chat usage (dict or SDK obj).
|
|
|
|
OpenAI-compatible backends with automatic prefix caching (vLLM, recent
|
|
llama-server) report the reused-prefix token count here; backends that don't
|
|
populate it yield 0.
|
|
"""
|
|
if not usage:
|
|
return 0
|
|
details = usage.get("prompt_tokens_details") if isinstance(usage, dict) \
|
|
else getattr(usage, "prompt_tokens_details", None)
|
|
if details is None:
|
|
return 0
|
|
if isinstance(details, dict):
|
|
return details.get("cached_tokens") or 0
|
|
return getattr(details, "cached_tokens", 0) or 0
|
|
|
|
|
|
def usage_chat_to_anthropic(usage, cache_read_tokens=0, cache_creation_tokens=0):
|
|
"""Map chat usage → Anthropic usage.
|
|
|
|
``cached_tokens`` reported by the backend (automatic prefix-cache reuse) plus any
|
|
explicit ``cache_read_tokens`` become ``cache_read_input_tokens``; ``input_tokens``
|
|
is the remaining uncached prompt so ``input + cache_read`` equals the prompt total.
|
|
|
|
``cache_creation_input_tokens`` stays at ``cache_creation_tokens`` (0 by default):
|
|
local backends do automatic KV-prefix reuse with no explicit ``cache_control``
|
|
write/breakpoint concept, so there is no honest "tokens written" figure to report —
|
|
that value is only real on the native Anthropic passthrough path.
|
|
"""
|
|
prompt = (usage or {}).get("prompt_tokens") or 0 if isinstance(usage, dict) \
|
|
else (getattr(usage, "prompt_tokens", 0) or 0)
|
|
completion = (usage or {}).get("completion_tokens") or 0 if isinstance(usage, dict) \
|
|
else (getattr(usage, "completion_tokens", 0) or 0)
|
|
cache_read = _cached_prompt_tokens(usage) + cache_read_tokens
|
|
return {
|
|
"input_tokens": max(prompt - cache_read, 0),
|
|
"output_tokens": completion,
|
|
"cache_creation_input_tokens": cache_creation_tokens,
|
|
"cache_read_input_tokens": cache_read,
|
|
}
|
|
|
|
|
|
def chat_message_to_content_blocks(message):
|
|
"""Convert an assistant chat message (dict) into Anthropic content blocks.
|
|
|
|
Ordering follows the Anthropic convention: thinking → text → tool_use.
|
|
"""
|
|
blocks = []
|
|
reasoning = message.get("reasoning_content") or message.get("reasoning")
|
|
if reasoning:
|
|
blocks.append({"type": "thinking", "thinking": reasoning})
|
|
content = message.get("content")
|
|
if content:
|
|
blocks.append({"type": "text", "text": content})
|
|
for tc in message.get("tool_calls") or []:
|
|
fn = tc.get("function", {})
|
|
try:
|
|
args = orjson.loads(fn.get("arguments") or "{}")
|
|
except (orjson.JSONDecodeError, TypeError):
|
|
args = {}
|
|
blocks.append({
|
|
"type": "tool_use",
|
|
"id": tc.get("id") or f"toolu_{secrets.token_hex(12)}",
|
|
"name": fn.get("name"),
|
|
"input": args,
|
|
})
|
|
return blocks
|
|
|
|
|
|
def build_message_object(*, message_id, model, content_blocks, stop_reason,
|
|
usage, stop_sequence=None):
|
|
"""Assemble a full ``type:"message"`` body for a non-streaming reply."""
|
|
return {
|
|
"id": message_id,
|
|
"type": "message",
|
|
"role": "assistant",
|
|
"model": model,
|
|
"content": content_blocks or [],
|
|
"stop_reason": stop_reason,
|
|
"stop_sequence": stop_sequence,
|
|
"usage": usage,
|
|
}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Cache-hit replay: a finished message object → Anthropic SSE stream
|
|
# ---------------------------------------------------------------------------
|
|
def _sse(etype, payload):
|
|
body = {"type": etype, **payload}
|
|
return f"event: {etype}\ndata: {orjson.dumps(body).decode('utf-8')}\n\n".encode("utf-8")
|
|
|
|
|
|
def message_object_to_sse(msg):
|
|
"""Render a *finished* message object as a valid Anthropic SSE event stream.
|
|
|
|
Used to serve cache hits to streaming clients without a backend call.
|
|
"""
|
|
out = []
|
|
start = {**msg, "content": [], "stop_reason": None, "stop_sequence": None,
|
|
"usage": {**(msg.get("usage") or {}), "output_tokens": 0}}
|
|
out.append(_sse("message_start", {"message": start}))
|
|
for i, block in enumerate(msg.get("content") or []):
|
|
btype = block.get("type")
|
|
if btype == "text":
|
|
out.append(_sse("content_block_start", {
|
|
"index": i, "content_block": {"type": "text", "text": ""}}))
|
|
out.append(_sse("content_block_delta", {
|
|
"index": i, "delta": {"type": "text_delta", "text": block.get("text", "")}}))
|
|
elif btype == "thinking":
|
|
out.append(_sse("content_block_start", {
|
|
"index": i, "content_block": {"type": "thinking", "thinking": ""}}))
|
|
out.append(_sse("content_block_delta", {
|
|
"index": i,
|
|
"delta": {"type": "thinking_delta", "thinking": block.get("thinking", "")}}))
|
|
elif btype == "tool_use":
|
|
out.append(_sse("content_block_start", {
|
|
"index": i,
|
|
"content_block": {"type": "tool_use", "id": block.get("id"),
|
|
"name": block.get("name"), "input": {}}}))
|
|
out.append(_sse("content_block_delta", {
|
|
"index": i,
|
|
"delta": {"type": "input_json_delta",
|
|
"partial_json": orjson.dumps(block.get("input") or {}).decode("utf-8")}}))
|
|
out.append(_sse("content_block_stop", {"index": i}))
|
|
out.append(_sse("message_delta", {
|
|
"delta": {"stop_reason": msg.get("stop_reason"),
|
|
"stop_sequence": msg.get("stop_sequence")},
|
|
"usage": {"output_tokens": (msg.get("usage") or {}).get("output_tokens", 0)}}))
|
|
out.append(_sse("message_stop", {}))
|
|
return b"".join(out)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Streaming direction: Chat Completions deltas → Anthropic typed SSE events
|
|
# ---------------------------------------------------------------------------
|
|
class ChatToMessagesStream:
|
|
"""Translate a Chat Completions streaming generator into Anthropic events.
|
|
|
|
Usage::
|
|
|
|
translator = ChatToMessagesStream(message_id, model)
|
|
async for sse_bytes in translator.events(chat_async_gen):
|
|
yield sse_bytes
|
|
# translator.content_blocks / usage / stop_reason now populated for storage
|
|
|
|
Emits ``message_start`` → (``content_block_start`` → ``*_delta``* →
|
|
``content_block_stop``)* → ``message_delta`` (stop_reason + usage) →
|
|
``message_stop``. A single monotonically increasing ``index`` is assigned across
|
|
thinking / text / tool_use blocks.
|
|
"""
|
|
|
|
def __init__(self, message_id, model, cache_read_tokens=0, cache_creation_tokens=0):
|
|
self.message_id = message_id
|
|
self.model = model
|
|
self.cache_read_tokens = cache_read_tokens
|
|
self.cache_creation_tokens = cache_creation_tokens
|
|
self.usage = None
|
|
self.stop_reason = "end_turn"
|
|
self.content_blocks = []
|
|
|
|
def _open_block(self, index, content_block):
|
|
return _sse("content_block_start", {"index": index, "content_block": content_block})
|
|
|
|
def _delta(self, index, delta):
|
|
return _sse("content_block_delta", {"index": index, "delta": delta})
|
|
|
|
def _stop(self, index):
|
|
return _sse("content_block_stop", {"index": index})
|
|
|
|
async def events(self, async_gen):
|
|
yield _sse("message_start", {"message": {
|
|
"id": self.message_id, "type": "message", "role": "assistant",
|
|
"model": self.model, "content": [], "stop_reason": None,
|
|
"stop_sequence": None,
|
|
"usage": {"input_tokens": 0, "output_tokens": 0,
|
|
"cache_creation_input_tokens": self.cache_creation_tokens,
|
|
"cache_read_input_tokens": self.cache_read_tokens},
|
|
}})
|
|
|
|
next_index = 0
|
|
# open block state: ("thinking"|"text", index) or None
|
|
open_kind = None
|
|
open_index = None
|
|
thinking_text = []
|
|
text_parts = []
|
|
finish_reason = None
|
|
# tool call state: chat tool-call index -> {index, id, name, args}
|
|
tc_state = {}
|
|
|
|
def close_open():
|
|
nonlocal open_kind, open_index
|
|
if open_kind is not None:
|
|
ev = self._stop(open_index)
|
|
open_kind, open_index = None, None
|
|
return ev
|
|
return None
|
|
|
|
async for chunk in async_gen:
|
|
usage = getattr(chunk, "usage", None)
|
|
if usage is not None:
|
|
self.usage = {
|
|
"prompt_tokens": getattr(usage, "prompt_tokens", 0) or 0,
|
|
"completion_tokens": getattr(usage, "completion_tokens", 0) or 0,
|
|
"prompt_tokens_details": {"cached_tokens": _cached_prompt_tokens(usage)},
|
|
}
|
|
choices = getattr(chunk, "choices", None)
|
|
if not choices:
|
|
continue
|
|
choice = choices[0]
|
|
if getattr(choice, "finish_reason", None):
|
|
finish_reason = choice.finish_reason
|
|
delta = choice.delta
|
|
|
|
reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None)
|
|
if reasoning:
|
|
if open_kind != "thinking":
|
|
ev = close_open()
|
|
if ev:
|
|
yield ev
|
|
open_kind, open_index = "thinking", next_index
|
|
next_index += 1
|
|
yield self._open_block(open_index, {"type": "thinking", "thinking": ""})
|
|
thinking_text.append(reasoning)
|
|
yield self._delta(open_index, {"type": "thinking_delta", "thinking": reasoning})
|
|
|
|
content_piece = getattr(delta, "content", None)
|
|
if content_piece:
|
|
if open_kind != "text":
|
|
ev = close_open()
|
|
if ev:
|
|
yield ev
|
|
open_kind, open_index = "text", next_index
|
|
next_index += 1
|
|
yield self._open_block(open_index, {"type": "text", "text": ""})
|
|
text_parts.append(content_piece)
|
|
yield self._delta(open_index, {"type": "text_delta", "text": content_piece})
|
|
|
|
for tc in getattr(delta, "tool_calls", None) or []:
|
|
idx = tc.index
|
|
fn = getattr(tc, "function", None)
|
|
if idx not in tc_state:
|
|
ev = close_open()
|
|
if ev:
|
|
yield ev
|
|
block_index = next_index
|
|
next_index += 1
|
|
state = {
|
|
"index": block_index,
|
|
"id": getattr(tc, "id", None) or f"toolu_{secrets.token_hex(12)}",
|
|
"name": (fn.name if fn else None),
|
|
"args": "",
|
|
}
|
|
tc_state[idx] = state
|
|
open_kind, open_index = "tool", block_index
|
|
yield self._open_block(block_index, {
|
|
"type": "tool_use", "id": state["id"], "name": state["name"], "input": {}})
|
|
else:
|
|
state = tc_state[idx]
|
|
if getattr(tc, "id", None):
|
|
state["id"] = tc.id
|
|
if fn and fn.name:
|
|
state["name"] = fn.name
|
|
if fn and fn.arguments:
|
|
state["args"] += fn.arguments
|
|
yield self._delta(state["index"], {
|
|
"type": "input_json_delta", "partial_json": fn.arguments})
|
|
|
|
ev = close_open()
|
|
if ev:
|
|
yield ev
|
|
|
|
# Assemble final content blocks (thinking → text → tool_use) for storage.
|
|
if thinking_text:
|
|
self.content_blocks.append({"type": "thinking", "thinking": "".join(thinking_text)})
|
|
if text_parts:
|
|
self.content_blocks.append({"type": "text", "text": "".join(text_parts)})
|
|
for idx in sorted(tc_state.keys()):
|
|
state = tc_state[idx]
|
|
try:
|
|
parsed = orjson.loads(state["args"]) if state["args"] else {}
|
|
except (orjson.JSONDecodeError, TypeError):
|
|
parsed = {}
|
|
self.content_blocks.append({
|
|
"type": "tool_use", "id": state["id"], "name": state["name"], "input": parsed})
|
|
|
|
self.stop_reason = finish_reason_to_stop_reason(finish_reason, has_tool_use=bool(tc_state))
|
|
final_usage = usage_chat_to_anthropic(
|
|
self.usage, cache_read_tokens=self.cache_read_tokens,
|
|
cache_creation_tokens=self.cache_creation_tokens)
|
|
yield _sse("message_delta", {
|
|
"delta": {"stop_reason": self.stop_reason, "stop_sequence": None},
|
|
"usage": {
|
|
"input_tokens": final_usage["input_tokens"],
|
|
"output_tokens": final_usage["output_tokens"],
|
|
"cache_read_input_tokens": final_usage["cache_read_input_tokens"],
|
|
"cache_creation_input_tokens": final_usage["cache_creation_input_tokens"],
|
|
}})
|
|
yield _sse("message_stop", {})
|