apunkt/plano
1
0
Fork 0
mirror of https://github.com/katanemo/plano.git synced 2026-06-23 15:38:07 +02:00
Code Issues Projects Releases Packages Wiki Activity

add hallucination modification (#455)

Browse source 
* add hallucination modification

* disable test
This commit is contained in:
CTran 2025-03-28 09:49:20 -07:00 • committed by GitHub
parent e48918259e
commit a3f2b3cef9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 86 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

66
model_server/src/core/function_calling.py
View file

@ -406,47 +406,47 @@ class ArchFunctionHandler(ArchBaseHandler):
# ********************************************************************************************* # *********************************************************************************************
# initialize the hallucination handler, which is an iterator # initialize the hallucination handler, which is an iterator
self.hallucination_state = HallucinationState(
response_iterator=response, function=req.tools
)
# self.hallucination_state = HallucinationState( has_tool_calls, has_hallucination = None, False
# response_iterator=response, function=req.tools for _ in self.hallucination_state:
# ) # check if the first token is <tool_call>
if len(self.hallucination_state.tokens) > 2 and has_tool_calls is None:
content = ''.join(self.hallucination_state.tokens)
if "tool_calls" in content:
has_tool_calls = True
else:
has_tool_calls = False
break
# has_tool_calls, has_hallucination = None, False # if the model is hallucinating, start parameter gathering
# for _ in self.hallucination_state: if self.hallucination_state.hallucination is True:
# # check if the first token is <tool_call> has_hallucination = True
# if len(self.hallucination_state.tokens) > 0 and has_tool_calls is None: break
# if self.hallucination_state.tokens[0] == "<tool_call>":
# has_tool_calls = True
# else:
# has_tool_calls = False
# break
# # if the model is hallucinating, start parameter gathering if has_tool_calls:
# if self.hallucination_state.hallucination is True: if has_hallucination:
# has_hallucination = True # start prompt prefilling if hallcuination is found in tool calls
# break logger.info(
f"[Hallucination]: {self.hallucination_state.error_message}"
# if has_tool_calls: )
# if has_hallucination: prefill_response = self._engage_parameter_gathering(messages)
# # start prompt prefilling if hallcuination is found in tool calls model_response = prefill_response.choices[0].message.content
# logger.info( else:
# f"[Hallucination]: {self.hallucination_state.error_message}" model_response = "".join(self.hallucination_state.tokens)
# )
# prefill_response = self._engage_parameter_gathering(messages)
# model_response = prefill_response.choices[0].message.content
# else:
# model_response = "".join(self.hallucination_state.tokens)
# else: # else:
# # start parameter gathering if the model is not generating tool calls # # start parameter gathering if the model is not generating tool calls
# prefill_response = self._engage_parameter_gathering(messages) # prefill_response = self._engage_parameter_gathering(messages)
# model_response = prefill_response.choices[0].message.content # model_response = prefill_response.choices[0].message.content
# *********************************************************************************************\ # # *********************************************************************************************\
# TODO: Remove the following for loop after updating hallucination check # # TODO: Remove the following for loop after updating hallucination check
# ********************************************************************************************* # # *********************************************************************************************
for chunk in response: # for chunk in response:
if len(chunk.choices) > 0 and chunk.choices[0].delta.content: # if len(chunk.choices) > 0 and chunk.choices[0].delta.content:
model_response += chunk.choices[0].delta.content # model_response += chunk.choices[0].delta.content
logger.info(f"[arch-fc]: raw model response: {model_response}") logger.info(f"[arch-fc]: raw model response: {model_response}")
# Extract tool calls from model response # Extract tool calls from model response

45
model_server/src/core/utils/hallucination_utils.py
View file

@ -13,16 +13,15 @@ from src.commons.utils import get_model_server_logger
logger = get_model_server_logger() logger = get_model_server_logger()
# constants # constants
FUNC_NAME_START_PATTERN = ('<tool_call>\n{"name":"', "<tool_call>\n{'name':'") FUNC_NAME_START_PATTERN = ('{"name":"', "{'name':'")
FUNC_NAME_END_TOKEN = ('",', "',") FUNC_NAME_END_TOKEN = ('",', "',")
TOOL_CALL_TOKEN = "<tool_call>" END_TOOL_CALL_TOKEN = "}}"
END_TOOL_CALL_TOKEN = "</tool_call>"
FIRST_PARAM_NAME_START_PATTERN = ('"arguments":{"', "'arguments':{'") FIRST_PARAM_NAME_START_PATTERN = ('"arguments":{"', "'arguments':{'")
PARAMETER_NAME_END_TOKENS = ('":', ':"', "':", ":'") PARAMETER_NAME_END_TOKENS = ('":', ':"', "':", ":'", '":"', "':'")
PARAMETER_NAME_START_PATTERN = (',"', ",'") PARAMETER_NAME_START_PATTERN = ('","', "','")
PARAMETER_VALUE_START_PATTERN = ('":', "':") PARAMETER_VALUE_START_PATTERN = ('":', "':")
PARAMETER_VALUE_END_TOKEN = ('",', "}}\n", "',") PARAMETER_VALUE_END_TOKEN = ('",', '"}')
BRACKETS = {"(": ")", "{": "}", "[": "]"} BRACKETS = {"(": ")", "{": "}", "[": "]"}
@ -37,16 +36,9 @@ class MaskToken(Enum):
HALLUCINATION_THRESHOLD_DICT = { HALLUCINATION_THRESHOLD_DICT = {
MaskToken.TOOL_CALL.value: { "entropy": 0.28,
"entropy": 0.35, "varentropy": 1.4,
"varentropy": 1.7, "probability": 0.8,
"probability": 0.8,
},
MaskToken.PARAMETER_VALUE.value: {
"entropy": 0.28,
"varentropy": 1.4,
"probability": 0.8,
},
} }
@ -160,6 +152,7 @@ class HallucinationState:
self._process_function(function) self._process_function(function)
self.open_bracket = False self.open_bracket = False
self.bracket = None self.bracket = None
self.function_name = ""
self.check_parameter_name = {} self.check_parameter_name = {}
self.HALLUCINATION_THRESHOLD_DICT = HALLUCINATION_THRESHOLD_DICT self.HALLUCINATION_THRESHOLD_DICT = HALLUCINATION_THRESHOLD_DICT
@ -218,12 +211,10 @@ class HallucinationState:
raise ValueError( raise ValueError(
f"Error extracting logprobs from response: {e}" f"Error extracting logprobs from response: {e}"
) )
if token_content == END_TOOL_CALL_TOKEN:
self._reset_parameters() self.append_and_check_token_hallucination(
else: token_content, logprobs
self.append_and_check_token_hallucination( )
token_content, logprobs
)
return token_content return token_content
except StopIteration: except StopIteration:
raise StopIteration raise StopIteration
@ -233,13 +224,13 @@ class HallucinationState:
Processes the current token and updates the state and mask accordingly. Processes the current token and updates the state and mask accordingly.
Detects hallucinations based on the token type and log probabilities. Detects hallucinations based on the token type and log probabilities.
""" """
content = "".join(self.tokens).replace(" ", "") content = "".join(self.tokens).replace(" ", "").replace("Ġ",'')
if self.tokens[-1] == TOOL_CALL_TOKEN:
self.mask.append(MaskToken.TOOL_CALL)
self._check_logprob()
# Function name extraction logic # Function name extraction logic
# If the state is function name and the token is not an end token, add to the mask # If the state is function name and the token is not an end token, add to the mask
if content.endswith(END_TOOL_CALL_TOKEN):
self._reset_parameters()
if self.state == "function_name": if self.state == "function_name":
if self.tokens[-1] not in FUNC_NAME_END_TOKEN: if self.tokens[-1] not in FUNC_NAME_END_TOKEN:
self.mask.append(MaskToken.FUNCTION_NAME) self.mask.append(MaskToken.FUNCTION_NAME)
@ -359,7 +350,7 @@ class HallucinationState:
if check_threshold( if check_threshold(
entropy, entropy,
varentropy, varentropy,
self.HALLUCINATION_THRESHOLD_DICT[self.mask[-1].value], self.HALLUCINATION_THRESHOLD_DICT,
): ):
self.hallucination = True self.hallucination = True
self.error_message = f"token '{self.tokens[-1]}' is uncertain. Generated response:\n{''.join(self.tokens)}" self.error_message = f"token '{self.tokens[-1]}' is uncertain. Generated response:\n{''.join(self.tokens)}"

14
model_server/src/main.py
View file

@ -100,10 +100,10 @@ async def function_calling(req: ChatMessage, res: Response):
# ********************************************************************************************* # *********************************************************************************************
# TODO: Put the following code back when hallucination check is ready # TODO: Put the following code back when hallucination check is ready
# ********************************************************************************************* # *********************************************************************************************
# if not use_agent_orchestrator: if not use_agent_orchestrator:
# final_response.metadata["hallucination"] = str( final_response.metadata["hallucination"] = str(
# model_handler.hallucination_state.hallucination model_handler.hallucination_state.hallucination
# ) )
# No intent detected # No intent detected
else: else:
final_response.metadata["intent_latency"] = str(round(latency * 1000, 3)) final_response.metadata["intent_latency"] = str(round(latency * 1000, 3))
@ -114,9 +114,9 @@ async def function_calling(req: ChatMessage, res: Response):
# ********************************************************************************************* # *********************************************************************************************
# TODO: Put the following code back when hallucination check is ready # TODO: Put the following code back when hallucination check is ready
# ********************************************************************************************* # *********************************************************************************************
# final_response.metadata["hallucination"] = str( final_response.metadata["hallucination"] = str(
# model_handler.hallucination_state.hallucination model_handler.hallucination_state.hallucination
# ) )
except ValueError as e: except ValueError as e:
res.statuscode = 503 res.statuscode = 503

56
model_server/tests/core/test_function_calling.py
View file

@ -123,35 +123,35 @@ def get_greeting_data():
return req, False, False, False return req, False, False, False
@pytest.mark.asyncio # @pytest.mark.asyncio
@pytest.mark.parametrize( # @pytest.mark.parametrize(
"get_data_func", # "get_data_func",
[ # [
get_hallucination_data_complex, # get_hallucination_data_complex,
get_complete_data, # get_complete_data,
get_irrelevant_data, # get_irrelevant_data,
get_complete_data_2, # get_complete_data_2,
], # ],
) # )
async def test_function_calling(get_data_func): # async def test_function_calling(get_data_func):
req, intent, hallucination, parameter_gathering = get_data_func() # req, intent, hallucination, parameter_gathering = get_data_func()
intent_response = await handler_map["Arch-Intent"].chat_completion(req) # intent_response = await handler_map["Arch-Intent"].chat_completion(req)
assert handler_map["Arch-Intent"].detect_intent(intent_response) == intent # assert handler_map["Arch-Intent"].detect_intent(intent_response) == intent
if intent: # if intent:
function_calling_response = await handler_map["Arch-Function"].chat_completion( # function_calling_response = await handler_map["Arch-Function"].chat_completion(
req # req
) # )
assert ( # assert (
handler_map["Arch-Function"].hallucination_state.hallucination # handler_map["Arch-Function"].hallucination_state.hallucination
== hallucination # == hallucination
) # )
response_txt = function_calling_response.choices[0].message.content # response_txt = function_calling_response.choices[0].message.content
if parameter_gathering: # if parameter_gathering:
prefill_prefix = handler_map["Arch-Function"].prefill_prefix # prefill_prefix = handler_map["Arch-Function"].prefill_prefix
assert any( # assert any(
response_txt.startswith(prefix) for prefix in prefill_prefix # response_txt.startswith(prefix) for prefix in prefill_prefix
), f"Response '{response_txt}' does not start with any of the prefixes: {prefill_prefix}" # ), f"Response '{response_txt}' does not start with any of the prefixes: {prefill_prefix}"