apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

small adjustment data gen (remove empty)

Browse source
This commit is contained in:
51616 2025-07-03 11:02:46 +00:00
parent f8a81a1eea
commit 1a03b01a52
4 changed files with 17 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

8
data/generate_fw_edu_qa_v2.py
View file

@ -101,14 +101,18 @@ def postprocess_qa_pairs(res_txt: str):
has_left_over = n_gen_pairs < len(questions) or n_gen_pairs < len(answers)
for i in range(n_gen_pairs):
response = answers[i].strip()
question = questions[i].strip()
if not response or not question:
print(f"Skipping empty question or answer at index {i}")
continue
if (not has_left_over) and (i == n_gen_pairs - 1):
response, skip = check_should_skip(response, vllm_model)
if skip:
print(f"Skipping due to missing stop string")
n_skips += 1
continue
out_q.append(questions[i].strip())
out_a.append(answers[i].strip())
out_q.append(question.strip())
out_a.append(response.strip())
print(f"Skipped {n_skips} responses due to missing stop strings")
return out_q, out_a

8
data/generate_fw_edu_qa_v2_repeat.py
View file

@ -108,14 +108,18 @@ def postprocess_qa_pairs(res_txt: str):
has_left_over = n_gen_pairs < len(questions) or n_gen_pairs < len(answers)
for i in range(n_gen_pairs):
response = answers[i].strip()
question = questions[i].strip()
if not response or not question:
print(f"Skipping empty question or answer at index {i}")
continue
if (not has_left_over) and (i == n_gen_pairs - 1):
response, skip = check_should_skip(response, vllm_model)
if skip:
print(f"Skipping due to missing stop string")
n_skips += 1
continue
out_q.append(questions[i].strip())
out_a.append(answers[i].strip())
out_q.append(question.strip())
out_a.append(response.strip())
print(f"Skipped {n_skips} responses due to missing stop strings")
return out_q, out_a

2
data/self_generate_qa.py
View file

@ -237,7 +237,7 @@ def self_generate(
print("=" * 80)
print(f"Generated {len(samples)} samples")
random.shuffle(samples)
# random.shuffle(samples)
# Save results
if not args.debug: