apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

more strip + '\n\n' between ctx and prompt (fix misalign tokenizer)

Browse source
This commit is contained in:
51616 2025-01-10 18:18:11 +00:00
parent 4d50ad84fa
commit 4664c58d4f
1 changed files with 6 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
hyperlora/data_utils.py
View file

@ -341,16 +341,16 @@ def convert_ctx_prompt_response_to_messages(
system_msg = ""
if "system_message" in example:
system_msg = example["system_message"]
system_msg = example["system_message"].strip()
user_msg = example["prompt"]
user_msg = example["prompt"].strip()
if add_ctx_to_chat:
user_msg = example["context"] + "\n" + user_msg
user_msg = example["context"].strip() + "\n\n" + user_msg.strip()
messages = [
{"role": "system", "content": system_msg},
{"role": "user", "content": user_msg},
{"role": "assistant", "content": example["response"]},
{"role": "system", "content": system_msg.strip()},
{"role": "user", "content": user_msg.strip()},
{"role": "assistant", "content": example["response"].strip()},
]
return dict(messages=messages)