apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

self-gen data misaligned fixed + niah reproducible + vllm-tokenizers version combo

Browse source
This commit is contained in:
51616 2025-10-08 05:17:50 +00:00
parent 39ed1b3d75
commit a074fd3305
8 changed files with 64 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data/self_generate_qa.py
View file

@ -233,7 +233,7 @@ def self_generate(
tk = get_tokenizer(args.vllm_model, train=True)
self_qa_intx_tokens = tk(SELF_QA_INTX, add_special_tokens=False)["input_ids"]
self_qa_intx_tokens = tk(SELF_QA_INTX, add_special_tokens=False)["input_ids"][1:]
if args.remove_qa_template:
self_qa_intx_tokens = tk("\n\n", add_special_tokens=False)["input_ids"]
n_self_qa_intx_tokens = len(self_qa_intx_tokens)