"""Translation between the Anthropic **Messages API** and **Chat Completions**. The router speaks Chat Completions to its local backends (Ollama, llama-server, llama-swap). To expose ``/v1/messages`` transparently on top of that, this module converts in both directions: * request: Anthropic ``system`` / ``messages`` / ``tools`` → chat ``messages`` / ``tools`` * response: chat ``choices[0].message`` → Anthropic ``content`` blocks * stream: chat completion deltas → Anthropic typed SSE events Pure functions / a stream-translator class — no I/O, mirroring ``requests/responses.py``. The native passthrough path (configured ``anthropic_endpoints``) does not use this module; it forwards the Anthropic wire format straight through. Reasoning: an inbound ``thinking`` block maps to the backend's ``reasoning_effort``; a backend that streams ``reasoning_content`` is surfaced back as Anthropic ``thinking`` content blocks / ``thinking_delta`` events. """ import secrets import orjson # --------------------------------------------------------------------------- # Request direction: Anthropic Messages → Chat Completions # --------------------------------------------------------------------------- def _system_to_text(system): """Flatten Anthropic ``system`` (string or text blocks) to a plain string.""" if system is None: return None if isinstance(system, str): return system if isinstance(system, list): parts = [] for b in system: if isinstance(b, dict) and b.get("type") == "text": parts.append(b.get("text", "")) elif isinstance(b, str): parts.append(b) return "\n\n".join(p for p in parts if p) or None return None def _image_block_to_chat(block): """Convert an Anthropic ``image`` block to an OpenAI ``image_url`` part (or None).""" src = block.get("source") or {} stype = src.get("type") if stype == "base64": media = src.get("media_type", "image/png") data = src.get("data", "") return {"type": "image_url", "image_url": {"url": f"data:{media};base64,{data}"}} if stype == "url" and src.get("url"): return {"type": "image_url", "image_url": {"url": src["url"]}} return None def _tool_result_content_to_str(content): """Flatten an Anthropic ``tool_result`` content (string or blocks) to a string.""" if content is None: return "" if isinstance(content, str): return content if isinstance(content, list): parts = [] for b in content: if isinstance(b, dict): if b.get("type") == "text": parts.append(b.get("text", "")) elif b.get("type") == "image": parts.append("[image]") else: parts.append(orjson.dumps(b).decode("utf-8")) else: parts.append(str(b)) return "\n".join(parts) return orjson.dumps(content).decode("utf-8") def _user_content_to_chat(content): """Split an Anthropic user ``content`` into ``(chat_content, tool_messages)``. ``tool_result`` blocks become standalone OpenAI ``role:"tool"`` messages; the remaining text/image parts collapse to a chat ``content`` (string or list). """ tool_messages = [] if content is None or isinstance(content, str): return content, tool_messages parts = [] for b in content if isinstance(content, list) else []: if not isinstance(b, dict): parts.append({"type": "text", "text": str(b)}) continue btype = b.get("type") if btype == "text": parts.append({"type": "text", "text": b.get("text", "")}) elif btype == "image": img = _image_block_to_chat(b) if img: parts.append(img) elif btype == "tool_result": tool_messages.append({ "role": "tool", "tool_call_id": b.get("tool_use_id"), "content": _tool_result_content_to_str(b.get("content")), }) # document / other blocks: no chat equivalent → skip if not parts: chat_content = None elif len(parts) == 1 and parts[0].get("type") == "text": chat_content = parts[0]["text"] else: chat_content = parts return chat_content, tool_messages def _assistant_content_to_chat(content): """Convert an Anthropic assistant ``content`` to ``(text_or_parts, tool_calls)``.""" if content is None or isinstance(content, str): return content, None text_parts = [] tool_calls = [] for b in content if isinstance(content, list) else []: if not isinstance(b, dict): continue btype = b.get("type") if btype == "text": text_parts.append(b.get("text", "")) elif btype == "tool_use": tool_calls.append({ "id": b.get("id"), "type": "function", "function": { "name": b.get("name"), "arguments": orjson.dumps(b.get("input") or {}).decode("utf-8"), }, }) # thinking blocks in history have no chat equivalent → drop text = "".join(text_parts) if text_parts else None return text, (tool_calls or None) def anthropic_messages_to_chat(system, messages): """Build a Chat Completions ``messages`` list from Anthropic ``system`` + ``messages``.""" chat = [] sys_text = _system_to_text(system) if sys_text: chat.append({"role": "system", "content": sys_text}) for m in messages or []: role = m.get("role") content = m.get("content") if role == "user": chat_content, tool_messages = _user_content_to_chat(content) # Anthropic packs tool results into a user turn; OpenAI wants each as a # separate tool message *before* any trailing user text. chat.extend(tool_messages) if chat_content is not None and chat_content != []: chat.append({"role": "user", "content": chat_content}) elif role == "assistant": text, tool_calls = _assistant_content_to_chat(content) msg = {"role": "assistant", "content": text} if tool_calls: msg["tool_calls"] = tool_calls chat.append(msg) else: chat.append({"role": role or "user", "content": content}) return chat def tools_anthropic_to_chat(tools): """Map Anthropic tool definitions → Chat Completions function tools.""" if not tools: return None out = [] for t in tools: if not isinstance(t, dict): continue # Server tools (web_search, …) carry a ``type`` but no ``input_schema`` — # local backends can't run them, so skip. if "input_schema" not in t and t.get("type") not in (None, "custom"): continue fn = {"name": t.get("name")} if t.get("description"): fn["description"] = t["description"] fn["parameters"] = t.get("input_schema") or {"type": "object", "properties": {}} out.append({"type": "function", "function": fn}) return out or None def tool_choice_anthropic_to_chat(tool_choice): """Map Anthropic ``tool_choice`` → Chat Completions ``tool_choice``.""" if not isinstance(tool_choice, dict): return None ttype = tool_choice.get("type") if ttype == "auto": return "auto" if ttype == "any": return "required" if ttype == "none": return "none" if ttype == "tool" and tool_choice.get("name"): return {"type": "function", "function": {"name": tool_choice["name"]}} return None def _thinking_to_reasoning_effort(thinking): """Map an Anthropic ``thinking`` config to an OpenAI ``reasoning_effort`` level.""" if not isinstance(thinking, dict): return None if thinking.get("type") == "disabled": return None budget = thinking.get("budget_tokens") if not isinstance(budget, int): return "medium" # adaptive / enabled without an explicit budget if budget < 2048: return "low" if budget < 8192: return "medium" return "high" def anthropic_to_chat_send_params(payload, chat_messages, model): """Assemble the Chat Completions request body from an Anthropic payload.""" send = {"messages": chat_messages, "model": model} if payload.get("max_tokens") is not None: send["max_tokens"] = payload["max_tokens"] opt = { "temperature": payload.get("temperature"), "top_p": payload.get("top_p"), "stop": payload.get("stop_sequences"), "tools": tools_anthropic_to_chat(payload.get("tools")), "tool_choice": tool_choice_anthropic_to_chat(payload.get("tool_choice")), "reasoning_effort": _thinking_to_reasoning_effort(payload.get("thinking")), } send.update({k: v for k, v in opt.items() if v is not None}) return send # --------------------------------------------------------------------------- # Response direction: Chat Completions → Anthropic Messages # --------------------------------------------------------------------------- def new_message_id(): return f"msg_{secrets.token_hex(24)}" _STOP_REASON_MAP = { "stop": "end_turn", "length": "max_tokens", "tool_calls": "tool_use", "function_call": "tool_use", "content_filter": "end_turn", } def finish_reason_to_stop_reason(finish_reason, has_tool_use=False): if has_tool_use: return "tool_use" return _STOP_REASON_MAP.get(finish_reason, "end_turn") def _cached_prompt_tokens(usage): """Read ``prompt_tokens_details.cached_tokens`` from a chat usage (dict or SDK obj). OpenAI-compatible backends with automatic prefix caching (vLLM, recent llama-server) report the reused-prefix token count here; backends that don't populate it yield 0. """ if not usage: return 0 details = usage.get("prompt_tokens_details") if isinstance(usage, dict) \ else getattr(usage, "prompt_tokens_details", None) if details is None: return 0 if isinstance(details, dict): return details.get("cached_tokens") or 0 return getattr(details, "cached_tokens", 0) or 0 def usage_chat_to_anthropic(usage, cache_read_tokens=0, cache_creation_tokens=0): """Map chat usage → Anthropic usage. ``cached_tokens`` reported by the backend (automatic prefix-cache reuse) plus any explicit ``cache_read_tokens`` become ``cache_read_input_tokens``; ``input_tokens`` is the remaining uncached prompt so ``input + cache_read`` equals the prompt total. ``cache_creation_input_tokens`` stays at ``cache_creation_tokens`` (0 by default): local backends do automatic KV-prefix reuse with no explicit ``cache_control`` write/breakpoint concept, so there is no honest "tokens written" figure to report — that value is only real on the native Anthropic passthrough path. """ prompt = (usage or {}).get("prompt_tokens") or 0 if isinstance(usage, dict) \ else (getattr(usage, "prompt_tokens", 0) or 0) completion = (usage or {}).get("completion_tokens") or 0 if isinstance(usage, dict) \ else (getattr(usage, "completion_tokens", 0) or 0) cache_read = _cached_prompt_tokens(usage) + cache_read_tokens return { "input_tokens": max(prompt - cache_read, 0), "output_tokens": completion, "cache_creation_input_tokens": cache_creation_tokens, "cache_read_input_tokens": cache_read, } def chat_message_to_content_blocks(message): """Convert an assistant chat message (dict) into Anthropic content blocks. Ordering follows the Anthropic convention: thinking → text → tool_use. """ blocks = [] reasoning = message.get("reasoning_content") or message.get("reasoning") if reasoning: blocks.append({"type": "thinking", "thinking": reasoning}) content = message.get("content") if content: blocks.append({"type": "text", "text": content}) for tc in message.get("tool_calls") or []: fn = tc.get("function", {}) try: args = orjson.loads(fn.get("arguments") or "{}") except (orjson.JSONDecodeError, TypeError): args = {} blocks.append({ "type": "tool_use", "id": tc.get("id") or f"toolu_{secrets.token_hex(12)}", "name": fn.get("name"), "input": args, }) return blocks def build_message_object(*, message_id, model, content_blocks, stop_reason, usage, stop_sequence=None): """Assemble a full ``type:"message"`` body for a non-streaming reply.""" return { "id": message_id, "type": "message", "role": "assistant", "model": model, "content": content_blocks or [], "stop_reason": stop_reason, "stop_sequence": stop_sequence, "usage": usage, } # --------------------------------------------------------------------------- # Cache-hit replay: a finished message object → Anthropic SSE stream # --------------------------------------------------------------------------- def _sse(etype, payload): body = {"type": etype, **payload} return f"event: {etype}\ndata: {orjson.dumps(body).decode('utf-8')}\n\n".encode("utf-8") def message_object_to_sse(msg): """Render a *finished* message object as a valid Anthropic SSE event stream. Used to serve cache hits to streaming clients without a backend call. """ out = [] start = {**msg, "content": [], "stop_reason": None, "stop_sequence": None, "usage": {**(msg.get("usage") or {}), "output_tokens": 0}} out.append(_sse("message_start", {"message": start})) for i, block in enumerate(msg.get("content") or []): btype = block.get("type") if btype == "text": out.append(_sse("content_block_start", { "index": i, "content_block": {"type": "text", "text": ""}})) out.append(_sse("content_block_delta", { "index": i, "delta": {"type": "text_delta", "text": block.get("text", "")}})) elif btype == "thinking": out.append(_sse("content_block_start", { "index": i, "content_block": {"type": "thinking", "thinking": ""}})) out.append(_sse("content_block_delta", { "index": i, "delta": {"type": "thinking_delta", "thinking": block.get("thinking", "")}})) elif btype == "tool_use": out.append(_sse("content_block_start", { "index": i, "content_block": {"type": "tool_use", "id": block.get("id"), "name": block.get("name"), "input": {}}})) out.append(_sse("content_block_delta", { "index": i, "delta": {"type": "input_json_delta", "partial_json": orjson.dumps(block.get("input") or {}).decode("utf-8")}})) out.append(_sse("content_block_stop", {"index": i})) out.append(_sse("message_delta", { "delta": {"stop_reason": msg.get("stop_reason"), "stop_sequence": msg.get("stop_sequence")}, "usage": {"output_tokens": (msg.get("usage") or {}).get("output_tokens", 0)}})) out.append(_sse("message_stop", {})) return b"".join(out) # --------------------------------------------------------------------------- # Streaming direction: Chat Completions deltas → Anthropic typed SSE events # --------------------------------------------------------------------------- class ChatToMessagesStream: """Translate a Chat Completions streaming generator into Anthropic events. Usage:: translator = ChatToMessagesStream(message_id, model) async for sse_bytes in translator.events(chat_async_gen): yield sse_bytes # translator.content_blocks / usage / stop_reason now populated for storage Emits ``message_start`` → (``content_block_start`` → ``*_delta``* → ``content_block_stop``)* → ``message_delta`` (stop_reason + usage) → ``message_stop``. A single monotonically increasing ``index`` is assigned across thinking / text / tool_use blocks. """ def __init__(self, message_id, model, cache_read_tokens=0, cache_creation_tokens=0): self.message_id = message_id self.model = model self.cache_read_tokens = cache_read_tokens self.cache_creation_tokens = cache_creation_tokens self.usage = None self.stop_reason = "end_turn" self.content_blocks = [] def _open_block(self, index, content_block): return _sse("content_block_start", {"index": index, "content_block": content_block}) def _delta(self, index, delta): return _sse("content_block_delta", {"index": index, "delta": delta}) def _stop(self, index): return _sse("content_block_stop", {"index": index}) async def events(self, async_gen): yield _sse("message_start", {"message": { "id": self.message_id, "type": "message", "role": "assistant", "model": self.model, "content": [], "stop_reason": None, "stop_sequence": None, "usage": {"input_tokens": 0, "output_tokens": 0, "cache_creation_input_tokens": self.cache_creation_tokens, "cache_read_input_tokens": self.cache_read_tokens}, }}) next_index = 0 # open block state: ("thinking"|"text", index) or None open_kind = None open_index = None thinking_text = [] text_parts = [] finish_reason = None # tool call state: chat tool-call index -> {index, id, name, args} tc_state = {} def close_open(): nonlocal open_kind, open_index if open_kind is not None: ev = self._stop(open_index) open_kind, open_index = None, None return ev return None async for chunk in async_gen: usage = getattr(chunk, "usage", None) if usage is not None: self.usage = { "prompt_tokens": getattr(usage, "prompt_tokens", 0) or 0, "completion_tokens": getattr(usage, "completion_tokens", 0) or 0, "prompt_tokens_details": {"cached_tokens": _cached_prompt_tokens(usage)}, } choices = getattr(chunk, "choices", None) if not choices: continue choice = choices[0] if getattr(choice, "finish_reason", None): finish_reason = choice.finish_reason delta = choice.delta reasoning = getattr(delta, "reasoning_content", None) or getattr(delta, "reasoning", None) if reasoning: if open_kind != "thinking": ev = close_open() if ev: yield ev open_kind, open_index = "thinking", next_index next_index += 1 yield self._open_block(open_index, {"type": "thinking", "thinking": ""}) thinking_text.append(reasoning) yield self._delta(open_index, {"type": "thinking_delta", "thinking": reasoning}) content_piece = getattr(delta, "content", None) if content_piece: if open_kind != "text": ev = close_open() if ev: yield ev open_kind, open_index = "text", next_index next_index += 1 yield self._open_block(open_index, {"type": "text", "text": ""}) text_parts.append(content_piece) yield self._delta(open_index, {"type": "text_delta", "text": content_piece}) for tc in getattr(delta, "tool_calls", None) or []: idx = tc.index fn = getattr(tc, "function", None) if idx not in tc_state: ev = close_open() if ev: yield ev block_index = next_index next_index += 1 state = { "index": block_index, "id": getattr(tc, "id", None) or f"toolu_{secrets.token_hex(12)}", "name": (fn.name if fn else None), "args": "", } tc_state[idx] = state open_kind, open_index = "tool", block_index yield self._open_block(block_index, { "type": "tool_use", "id": state["id"], "name": state["name"], "input": {}}) else: state = tc_state[idx] if getattr(tc, "id", None): state["id"] = tc.id if fn and fn.name: state["name"] = fn.name if fn and fn.arguments: state["args"] += fn.arguments yield self._delta(state["index"], { "type": "input_json_delta", "partial_json": fn.arguments}) ev = close_open() if ev: yield ev # Assemble final content blocks (thinking → text → tool_use) for storage. if thinking_text: self.content_blocks.append({"type": "thinking", "thinking": "".join(thinking_text)}) if text_parts: self.content_blocks.append({"type": "text", "text": "".join(text_parts)}) for idx in sorted(tc_state.keys()): state = tc_state[idx] try: parsed = orjson.loads(state["args"]) if state["args"] else {} except (orjson.JSONDecodeError, TypeError): parsed = {} self.content_blocks.append({ "type": "tool_use", "id": state["id"], "name": state["name"], "input": parsed}) self.stop_reason = finish_reason_to_stop_reason(finish_reason, has_tool_use=bool(tc_state)) final_usage = usage_chat_to_anthropic( self.usage, cache_read_tokens=self.cache_read_tokens, cache_creation_tokens=self.cache_creation_tokens) yield _sse("message_delta", { "delta": {"stop_reason": self.stop_reason, "stop_sequence": None}, "usage": { "input_tokens": final_usage["input_tokens"], "output_tokens": final_usage["output_tokens"], "cache_read_input_tokens": final_usage["cache_read_input_tokens"], "cache_creation_input_tokens": final_usage["cache_creation_input_tokens"], }}) yield _sse("message_stop", {})