apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-26 17:11:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

add chat_template for instruct ctx encoder

Browse source
This commit is contained in:
51616 2025-01-27 10:11:20 +00:00
parent 3cb527a337
commit 74e54adf0e
1 changed files with 13 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

16
src/ctx_to_lora/data_utils.py
View file

@ -719,11 +719,21 @@ def tokenize_chat_messages(
def tokenize_ctx_text(
example: dict[str, Any],
samples: dict[str, Any],
tokenizer: PreTrainedTokenizerBase,
) -> dict[str, Any]:
text = example["context"]
tokenized_text = tokenizer(text)
if tokenizer.chat_template:
txt = tokenizer.apply_chat_template(
[[{"role": "user", "content": text.strip()}] for text in samples["context"]],
tokenize=False,
add_generation_prompt=False,
)
# special tokens are already added by the chat template
tokenized_text = tokenizer(txt, truncation=False, add_special_tokens=False)
else:
txt = samples["context"]
tokenized_text = tokenizer(txt, truncation=False)
ctx_ids = tokenized_text["input_ids"]
ctx_attn_mask = tokenized_text["attention_mask"]
return dict(ctx_ids=ctx_ids, ctx_attn_mask=ctx_attn_mask)