apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

compact script update

Browse source
This commit is contained in:
51616 2025-07-21 09:04:53 +00:00
parent c7ab8d05b3
commit 0945adeee5
2 changed files with 3 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data/build_drop_compact.py
View file

@ -17,7 +17,7 @@ if __name__ == "__main__":
if ctx not in ctx_qa_dict:
ctx_qa_dict[ctx] = {"prompts": [], "responses": []}
question = closed_qa_prompting(sample["question"])
answer = ", ".join(set(sample["answers_spans"]["spans"]))
answer = sample["answers_spans"]["spans"][0]
ctx_qa_dict[ctx]["prompts"].append(question)
ctx_qa_dict[ctx]["responses"].append(answer)

5
data/build_pwc_compact.py
View file

@ -3,8 +3,6 @@ import gc
from datasets import Dataset, load_dataset
from tqdm import tqdm
from ctx_to_lora.data.processing import closed_qa_prompting
if __name__ == "__main__":
ds_name = "sggetao/PwC"
@ -16,7 +14,8 @@ if __name__ == "__main__":
ctx = sample["input"]
if ctx not in ctx_qa_dict:
ctx_qa_dict[ctx] = {"prompts": [], "responses": []}
question = closed_qa_prompting(sample["prompt"])
# question = closed_qa_prompting(sample["prompt"])
question = sample["prompt"]
answer = sample["answer"]
ctx_qa_dict[ctx]["prompts"].append(question)
ctx_qa_dict[ctx]["responses"].append(answer)