apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-26 17:11:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

update data gen/def

Browse source
This commit is contained in:
51616 2025-09-06 08:43:36 +00:00
parent cfe737d106
commit 0c1a8bb3e8
5 changed files with 76 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

4
data/build_squad_compact.py
View file

@ -3,8 +3,6 @@ import gc
from datasets import Dataset, load_dataset
from tqdm import tqdm
from ctx_to_lora.data.preprocessing_fn import closed_qa_prompting
if __name__ == "__main__":
ds_name = "data/raw_datasets/squad"
@ -16,7 +14,7 @@ if __name__ == "__main__":
ctx = sample["context"]
if ctx not in ctx_qa_dict:
ctx_qa_dict[ctx] = {"prompts": [], "responses": []}
question = closed_qa_prompting(sample["question"])
question = sample["question"]
answer = sample["answers"]["text"][0]
ctx_qa_dict[ctx]["prompts"].append(question)
ctx_qa_dict[ctx]["responses"].append(answer)