apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

vlm ctx-encoder + slrum scripts + qwen self-gen

Browse source
This commit is contained in:
51616 2025-10-27 06:20:26 +00:00
parent 178822b7d3
commit 2ed418bc22
13 changed files with 495 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

4
data/self_generate_qa.py
View file

@ -38,6 +38,8 @@ MODEL_CTX_LEN = {
"google/gemma-2-27b-it": 8192,
"google/gemma-2-2b-it": 8192,
"google/gemma-2-9b-it": 8192,
# qwen 4b has 256k ctx length but using lower max lengths is faster
"Qwen/Qwen3-4B-Instruct-2507": 2**13 + 2**12,
}
@ -241,7 +243,7 @@ def self_generate(
n_pre_ctx_tokens = len(pre_ctx_tokens)
sys_tokens = tk(system_template.split("\n")[0], add_special_tokens=False)[
"input_ids"
]
][:-1]
n_sys_tokens = len(sys_tokens)
os.environ["TOKENIZERS_PARALLELISM"] = "true"
ds = ds.map(