from glob import glob IGNORE_INDEX = -100 FW_QA_PATHS = [ f"data/raw_datasets/fw_qa/{i:05d}.parquet" for i in [0, 1, 6, 7, 8, 10, 22, 30, 35] ] TRANSFORMED_DATA_DIR = "data/processed_datasets" # approximate length of the datasets (train split) # needed for streaming datasets DS_LEN = { "fw_qa_3_mini": 100_000, "fw_qa_3_medium": 121_000_000, "fw_qa_3": 270_000_000, "fw_qa_xl": 27_000_000, "ctx_qa": 300_000, "pwc": 240_000, "hotpot_qa": 100_000, "squad": 90_000, "drop": 77_000, "narrativeqa": 40_000, "quoref": 11_000, "ropes": 11_000, "synthetic_convqa": 40_000, } LONGBENCH_TASKS = [ "longbench/narrativeqa", "longbench/qasper", "longbench/multifieldqa_en", "longbench/multifieldqa_zh", "longbench/hotpotqa", "longbench/2wikimqa", "longbench/musique", "longbench/dureader", "longbench/gov_report", "longbench/qmsum", "longbench/multi_news", "longbench/vcsum", ] LONGBENCH_E_TASKS = [ "longbench/qasper_e", "longbench/multifieldqa_en_e", "longbench/hotpotqa_e", "longbench/2wikimqa_e", "longbench/gov_report_e", "longbench/multi_news_e", ] DS_KWARGS = { "hotpot_qa": dict( train=dict(path="hotpotqa/hotpot_qa", name="fullwiki", split="train[900:]"), validation=dict(path="hotpotqa/hotpot_qa", name="fullwiki", split="train[:900]"), test=dict(path="hotpotqa/hotpot_qa", name="fullwiki", split="validation"), ), "hotpot_qa_tiny": dict( train=dict(path="hotpotqa/hotpot_qa", name="fullwiki", split="train[900:2000]"), validation=dict(path="hotpotqa/hotpot_qa", name="fullwiki", split="train[:900]"), ), "pwc": dict( train=dict(path="sggetao/PwC", split="train[900:]"), validation=dict(path="sggetao/PwC", split="train[:900]"), test=dict(path="sggetao/PwC", split="test"), ), "pwc_tiny": dict( train=dict(path="sggetao/PwC", split="train[900:2000]"), validation=dict(path="sggetao/PwC", split="train[:900]"), ), "squad": dict( train=dict(path="rajpurkar/squad", split="train[900:]"), validation=dict(path="rajpurkar/squad", split="train[:900]"), test=dict(path="rajpurkar/squad", split="validation"), ), # "fw_qa_tiny": dict( # train=dict( # path="parquet", # data_files="data/raw_datasets/fw_qa/00000.parquet", # split="train", # ), # validation=dict( # path="parquet", # data_files="data/raw_datasets/fw_qa/00000_val.parquet", # split="train", # ), # ), # "fw_qa": dict( # train=dict( # path="parquet", # data_files=FW_QA_PATHS, # split="train", # ), # validation=dict( # path="parquet", # data_files="data/raw_datasets/fw_qa/*_val.parquet", # split="train", # ), # ), # "fw_qa_large": dict( # train=dict( # path="parquet", # data_files=glob("data/raw_datasets/fw_qa_large/*res.parquet"), # split="train", # ), # validation=dict( # path="parquet", # data_files=glob("data/raw_datasets/fw_qa_large/*val.parquet"), # split="train", # ), # ), "fw_qa_xl": dict( train=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_xl/*[!val].parquet"), split="train", ), validation=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_xl/*val.parquet"), split="train", ), # there is no explcit test for this ds # used only for sanity check test=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_xl/*val.parquet"), split="train", ), ), # "fw_qa_2": dict( # train=dict( # path="parquet", # data_files=glob("data/raw_datasets/fw_qa_2/*[!val].parquet"), # split="train", # ), # validation=dict( # path="parquet", # data_files=glob("data/raw_datasets/fw_qa_2/*val.parquet"), # split="train", # ), # ), "fw_qa_3_mini": dict( train=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_3/000_00001.parquet"), split="train[:100000]", ), ), "fw_qa_3_medium": dict( train=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_3/00[0-5]*[!val].parquet"), split="train", ), ), "fw_qa_3": dict( train=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_3/*[!val].parquet"), split="train", ), validation=dict( path="parquet", data_files=glob("data/raw_datasets/fw_qa_3/*val.parquet"), split="train", ), ), "ctx_qa": dict( train=dict( path="parquet", data_files=glob("data/raw_datasets/ctx_qa/*res.parquet"), split="train", ), validation=dict( path="parquet", data_files=glob("data/raw_datasets/ctx_qa/*val.parquet"), split="train", ), # there is no explcit test for this ds # used only for sanity check test=dict( path="parquet", data_files=glob("data/raw_datasets/ctx_qa/*val.parquet"), split="train", ), ), "drop": dict( train=dict( path="ucinlp/drop", split="train", ), ), "narrativeqa": dict( train=dict( path="nvidia/ChatQA-Training-Data", name="narrativeqa", split="train", ), ), "quoref": dict( train=dict( path="nvidia/ChatQA-Training-Data", name="quoref", split="train", ), ), "ropes": dict( train=dict( path="nvidia/ChatQA-Training-Data", name="ropes", split="train", ), ), "synthetic_convqa": dict( train=dict( path="nvidia/ChatQA-Training-Data", name="synthetic_convqa", split="train", ), ), "booksum": dict( train=dict( path="kmfoda/booksum", split="train", ), validation=dict( path="kmfoda/booksum", split="validation", ), test=dict( path="kmfoda/booksum", split="test", ), ), "gov_report": dict( train=dict( path="ccdv/govreport-summarization", split="train", ), validation=dict( path="ccdv/govreport-summarization", split="validation", ), test=dict( path="ccdv/govreport-summarization", split="test", ), ), # "wikitext-2": dict( # train=dict( # path="EleutherAI/wikitext_document_level", # name="wikitext-2-raw-v1", # split="train", # ), # ), # "wikitext-103": dict( # train=dict( # path="EleutherAI/wikitext_document_level", # name="wikitext-103-raw-v1", # split="train", # ), # ), "gsm8k": dict( train=dict( path="openai/gsm8k", name="main", split="train[100:]", ), validation=dict( path="openai/gsm8k", name="main", split="train[:100]", ), ), "openmathintx-2": dict( train=dict( path="nvidia/OpenMathInstruct-2", split="train_1M[:100000]", ), ), "opencoder-edu": dict( train=dict( path="OpenCoder-LLM/opc-sft-stage2", name="educational_instruct", split="train[900:]", ), validation=dict( path="OpenCoder-LLM/opc-sft-stage2", name="educational_instruct", split="train[:900]", ), ), "triviaqa_retrieved": dict( test=dict( path="json", data_files="data/eval/triviaqa/triviaqa_retrieved.jsonl", split="train", ), ), # for negative training and evaluating "negative_nq": dict( test=dict( path="json", data_files="data/raw_datasets/negative_natural_questions/validation.jsonl", split="train", ), ), } # LongBench kwargs for ds_name in LONGBENCH_TASKS + LONGBENCH_E_TASKS: DS_KWARGS[ds_name] = dict( test=dict( path="THUDM/LongBench", name=ds_name.split("/")[-1], split="test", ) ) # for training closed qa datasets, e.g., hotpot_qa, squad, etc. CLOSED_QA_INTX_TEMPLATES = [ "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}", "Answer without any explanation.\n\nQuestion: {input}", "Based on the provided text, what is the answer to the following question? Provide only the answer.\n\nQuestion: {input}", "Extract the answer to the question from the text. Be concise. Do not explain.\n\nQuestion: {input}", "What is the answer to this question, based on the context? Respond with the answer only.\n\nQuestion: {input}", "Provide a direct answer to the question using the given passages. Do not give any explanation.\n\nQuestion: {input}", "Answer the question using only information from the provided text. No extra words.\n\nQuestion: {input}", "From the passages, answer the question. Just the answer, please.\n\nQuestion: {input}", "Give the answer to the question. Do not include any other text.\n\nQuestion: {input}", "The answer to the question is in the text. Find it and state it clearly. No need for explanation.\n\nQuestion: {input}", "Concisely answer the question based on the text provided. Don't include any other words. Just the answer.\n\nQuestion: {input}", "Read the passages and answer the question with the minimal necessary words.\n\nQuestion: {input}", "What is the direct response to the question, according to the text? Avoid explanation.\n\nQuestion: {input}", "Please provide only the answer to the question, derived from the text.\n\nQuestion: {input}", "Using the provided context, answer the question. Output the answer and nothing else.\n\nQuestion: {input}", "Identify the answer in the text and present it without elaboration.\n\nQuestion: {input}", "Answer the following question based on the text. Your answer should be brief and to the point. No explanation.\n\nQuestion: {input}", "Based on the information given, what is the answer to the question? Only state the answer.\n\nQuestion: {input}", "Find the answer to the question in the provided passages and write it down. No explanations.\n\nQuestion: {input}", "The question is: {input}. Provide the answer based on the text, and nothing more.", "Question: {input}\nAnswer directly based on the text provided. No extra words.", "Question: {input}\nPlease provide the answer based on the text. No explanation is needed.", ] EVAL_INTX_TEMPLATES = { "negative_nq": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", "triviaqa_retrieved": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", "hotpot_qa": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", "squad": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", "longbench/qasper": 'Answer the question as concisely as you can, using a single phrase or sentence if possible.\nIf the question cannot be answered based on the information in the article, write "unanswerable".\nIf the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {input}', "longbench/narrativeqa": "Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}", "longbench/multifieldqa_en": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", # "longbench/multifieldqa_zh": "阅读以下文字并用中文简短回答：\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}", "longbench/hotpotqa": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", "longbench/2wikimqa": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", "longbench/musique": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", # # "longbench/dureader": "请基于给定的文章回答下述问题\n\n请基于上述文章回答下面的问题。\n\n问题：{input}", "longbench/gov_report": "You are given a report by a government agency. Write a one-page summary of the report.", "longbench/qmsum": "You are given a meeting transcript and a query containing a question or instruction. Answer the query based on the above meeting transcript.\n\nQuery: {input}", "longbench/multi_news": "You are given several news passages. Write a one-page summary of all the news.", # "longbench/vcsum": "下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。", } for ds_name in LONGBENCH_E_TASKS: EVAL_INTX_TEMPLATES[ds_name] = EVAL_INTX_TEMPLATES[ds_name[:-2]] for ds_name in DS_KWARGS: if ds_name not in EVAL_INTX_TEMPLATES: EVAL_INTX_TEMPLATES[ds_name] = "{input}"