From 4664c58d4f9b230168cb9d57546d59b5e3b4ef7f Mon Sep 17 00:00:00 2001 From: 51616 Date: Fri, 10 Jan 2025 18:18:11 +0000 Subject: [PATCH] more strip + '\n\n' between ctx and prompt (fix misalign tokenizer) --- hyperlora/data_utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/hyperlora/data_utils.py b/hyperlora/data_utils.py index 42dd5e8..d61640c 100644 --- a/hyperlora/data_utils.py +++ b/hyperlora/data_utils.py @@ -341,16 +341,16 @@ def convert_ctx_prompt_response_to_messages( system_msg = "" if "system_message" in example: - system_msg = example["system_message"] + system_msg = example["system_message"].strip() - user_msg = example["prompt"] + user_msg = example["prompt"].strip() if add_ctx_to_chat: - user_msg = example["context"] + "\n" + user_msg + user_msg = example["context"].strip() + "\n\n" + user_msg.strip() messages = [ - {"role": "system", "content": system_msg}, - {"role": "user", "content": user_msg}, - {"role": "assistant", "content": example["response"]}, + {"role": "system", "content": system_msg.strip()}, + {"role": "user", "content": user_msg.strip()}, + {"role": "assistant", "content": example["response"].strip()}, ] return dict(messages=messages)