add bolt support (#90)

* add support for bolt

* improve logging

* add support for bolt-fc

* fix int tests
This commit is contained in:
Adil Hafeez 2024-09-26 17:47:01 -07:00 committed by GitHub
parent e3a835e5d3
commit 774c389951
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
5 changed files with 253 additions and 10 deletions

View file

@ -2,6 +2,7 @@ import json
import random
from fastapi import FastAPI, Response
from arch_handler import ArchHandler
from bolt_handler import BoltHandler
from common import ChatMessage
import logging
from openai import OpenAI
@ -11,13 +12,17 @@ ollama_endpoint = os.getenv("OLLAMA_ENDPOINT", "localhost")
ollama_model = os.getenv("OLLAMA_MODEL", "Arch-Function-Calling-1.5B-Q4_K_M")
logger = logging.getLogger('uvicorn.error')
handler = None
if ollama_model.startswith("Arch"):
handler = ArchHandler()
else:
handler = BoltHandler()
logger.info(f"using model: {ollama_model}")
logger.info(f"using ollama endpoint: {ollama_endpoint}")
app = FastAPI()
handler = ArchHandler()
client = OpenAI(
base_url='http://{}:11434/v1/'.format(ollama_endpoint),