mirror of
https://github.com/katanemo/plano.git
synced 2026-04-28 02:23:56 +02:00
50 lines
1.5 KiB
Python
50 lines
1.5 KiB
Python
import json
|
|
from fastapi import FastAPI, Response
|
|
from bolt_handler import BoltHandler
|
|
from arch_handler import ArchHandler
|
|
from common import ChatMessage
|
|
import logging
|
|
from openai import OpenAI
|
|
import os
|
|
|
|
ollama_endpoint = os.getenv("OLLAMA_ENDPOINT", "localhost")
|
|
ollama_model = os.getenv("OLLAMA_MODEL", "Bolt-Function-Calling-1B:Q4_K_M")
|
|
logger = logging.getLogger('uvicorn.error')
|
|
|
|
logger.info(f"using model: {ollama_model}")
|
|
logger.info(f"using ollama endpoint: {ollama_endpoint}")
|
|
|
|
app = FastAPI()
|
|
bolt_handler = BoltHandler()
|
|
arch_handler = ArchHandler()
|
|
|
|
client = OpenAI(
|
|
base_url='http://{}:11434/v1/'.format(ollama_endpoint),
|
|
|
|
# required but ignored
|
|
api_key='ollama',
|
|
)
|
|
|
|
@app.get("/healthz")
|
|
async def healthz():
|
|
return {
|
|
"status": "ok"
|
|
}
|
|
|
|
|
|
@app.post("/v1/chat/completions")
|
|
async def chat_completion(req: ChatMessage, res: Response):
|
|
logger.info("starting request")
|
|
if ollama_model.startswith("Bolt"):
|
|
handler = bolt_handler
|
|
else:
|
|
handler = arch_handler
|
|
tools_encoded = handler._format_system(req.tools)
|
|
# append system prompt with tools to messages
|
|
messages = [{"role": "system", "content": tools_encoded}]
|
|
for message in req.messages:
|
|
messages.append({"role": message.role, "content": message.content})
|
|
logger.info(f"request model: {ollama_model}, messages: {json.dumps(messages)}")
|
|
resp = client.chat.completions.create(messages=messages, model=ollama_model, stream=False)
|
|
logger.info(f"response: {resp.to_json()}")
|
|
return resp
|