diff --git a/model_server/src/core/function_calling.py b/model_server/src/core/function_calling.py index 7740cb46..59ba57b9 100644 --- a/model_server/src/core/function_calling.py +++ b/model_server/src/core/function_calling.py @@ -377,7 +377,7 @@ class ArchFunctionHandler(ArchBaseHandler): has_tool_calls, has_hallucination = None, False for _ in self.hallucination_state: # check if moodel response starts with tool calls - if has_tool_calls is None: + if len(self.hallucination_state.tokens)>5 and has_tool_calls is None: content = "".join(self.hallucination_state.tokens) if "tool_calls" in content: has_tool_calls = True @@ -400,13 +400,9 @@ class ArchFunctionHandler(ArchBaseHandler): stream=False, extra_body=self.generation_params, ) - model_response = ( - self.clarify_prefix + response.choices[0].message.content - ) + model_response = response.choices[0].message.content else: - model_response = self.default_prefix + "".join( - self.hallucination_state.tokens - ) + model_response = "".join(self.hallucination_state.tokens) # Extract tool calls from model response raw_model_response_json_fixed, response_dict = self._parse_model_response( diff --git a/model_server/src/core/utils/hallucination_utils.py b/model_server/src/core/utils/hallucination_utils.py index e2ee1667..51216c12 100644 --- a/model_server/src/core/utils/hallucination_utils.py +++ b/model_server/src/core/utils/hallucination_utils.py @@ -201,20 +201,20 @@ class HallucinationState: r = next(self.response_iterator) if hasattr(r.choices[0].delta, "content"): token_content = r.choices[0].delta.content - if token_content: + if token_content != '': try: logprobs = [ p.logprob for p in r.choices[0].logprobs.content[0].top_logprobs ] + self.append_and_check_token_hallucination( + token_content, logprobs + ) except Exception as e: - raise ValueError( - f"Error extracting logprobs from response: {e}" + self.append_and_check_token_hallucination( + token_content, [None] ) - self.append_and_check_token_hallucination( - token_content, logprobs - ) return token_content except StopIteration: raise StopIteration