diff --git a/data/build_drop_compact.py b/data/build_drop_compact.py index d126c71..a8e9bfd 100644 --- a/data/build_drop_compact.py +++ b/data/build_drop_compact.py @@ -17,7 +17,7 @@ if __name__ == "__main__": if ctx not in ctx_qa_dict: ctx_qa_dict[ctx] = {"prompts": [], "responses": []} question = closed_qa_prompting(sample["question"]) - answer = ", ".join(set(sample["answers_spans"]["spans"])) + answer = sample["answers_spans"]["spans"][0] ctx_qa_dict[ctx]["prompts"].append(question) ctx_qa_dict[ctx]["responses"].append(answer) diff --git a/data/build_pwc_compact.py b/data/build_pwc_compact.py index 63e4f72..b2c3465 100644 --- a/data/build_pwc_compact.py +++ b/data/build_pwc_compact.py @@ -3,8 +3,6 @@ import gc from datasets import Dataset, load_dataset from tqdm import tqdm -from ctx_to_lora.data.processing import closed_qa_prompting - if __name__ == "__main__": ds_name = "sggetao/PwC" @@ -16,7 +14,8 @@ if __name__ == "__main__": ctx = sample["input"] if ctx not in ctx_qa_dict: ctx_qa_dict[ctx] = {"prompts": [], "responses": []} - question = closed_qa_prompting(sample["prompt"]) + # question = closed_qa_prompting(sample["prompt"]) + question = sample["prompt"] answer = sample["answer"] ctx_qa_dict[ctx]["prompts"].append(question) ctx_qa_dict[ctx]["responses"].append(answer)