make context length configurable

This commit is contained in:
seehi 2024-10-28 21:12:09 +08:00
parent 8b209d4e17
commit 7b81c8a690
2 changed files with 4 additions and 1 deletions

View file

@ -83,6 +83,7 @@ class LLMConfig(YamlModel):
logprobs: Optional[bool] = None
top_logprobs: Optional[int] = None
timeout: int = 600
context_length: Optional[int] = None # Max input tokens
# For Amazon Bedrock
region_name: str = None