apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix empty q_list

Browse source
This commit is contained in:
51616 2025-08-29 02:36:00 +00:00
parent ee9039295e
commit a269aff51d
1 changed files with 1 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

3
data/self_generate_qa.py
View file

@ -322,7 +322,6 @@ def self_generate(
ds = ds.take(50)
ds = ds.filter(filter_none, batched=False, num_proc=8)
print(f"0. {len(ds)=}")
tk = get_tokenizer(args.vllm_model, train=True)
@ -363,7 +362,7 @@ def self_generate(
ctxs = [sample["context"] for sample in ds]
questions = ds["prompts"]
questions = [q_list for q_list in ds["prompts"] if len(q_list) > 0]
# for sample in ds:
# if ("type" not in sample) or (
# "type" in sample and sample["type"] == "question"