apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-17 15:25:17 +02:00
Code Issues Projects Releases Packages Wiki Activity

fix bugs when no logprob for prefill and bug in function calling loop when it always no tool call

Browse source
This commit is contained in:
co tran 2025-04-01 20:18:56 +00:00
parent a3ab6be51c
commit 1b39ee3dd8
2 changed files with 9 additions and 13 deletions
Download patch file Download diff file Expand all files Collapse all files

10
model_server/src/core/function_calling.py
View file

@ -377,7 +377,7 @@ class ArchFunctionHandler(ArchBaseHandler):
has_tool_calls, has_hallucination = None, False
for _ in self.hallucination_state:
# check if moodel response starts with tool calls
if has_tool_calls is None:
if len(self.hallucination_state.tokens)>5 and has_tool_calls is None:
content = "".join(self.hallucination_state.tokens)
if "tool_calls" in content:
has_tool_calls = True
@ -400,13 +400,9 @@ class ArchFunctionHandler(ArchBaseHandler):
stream=False,
extra_body=self.generation_params,
)
model_response = (
self.clarify_prefix + response.choices[0].message.content
)
model_response = response.choices[0].message.content
else:
model_response = self.default_prefix + "".join(
self.hallucination_state.tokens
)
model_response = "".join(self.hallucination_state.tokens)
# Extract tool calls from model response
raw_model_response_json_fixed, response_dict = self._parse_model_response(

12
model_server/src/core/utils/hallucination_utils.py
View file

@ -201,20 +201,20 @@ class HallucinationState:
r = next(self.response_iterator)
if hasattr(r.choices[0].delta, "content"):
token_content = r.choices[0].delta.content
if token_content:
if token_content != '':
try:
logprobs = [
p.logprob
for p in r.choices[0].logprobs.content[0].top_logprobs
]
self.append_and_check_token_hallucination(
token_content, logprobs
)
except Exception as e:
raise ValueError(
f"Error extracting logprobs from response: {e}"
self.append_and_check_token_hallucination(
token_content, [None]
)
self.append_and_check_token_hallucination(
token_content, logprobs
)
return token_content
except StopIteration:
raise StopIteration