mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
eval prompts + no context
This commit is contained in:
51616
parent
7f775b2fdf
commit
19ca0f99a5
3 changed files with 52 additions and 24 deletions
|
|
@ -341,21 +341,22 @@ CLOSED_QA_INTX_TEMPLATES = [
|
|||
|
||||
|
||||
EVAL_INTX_TEMPLATES = {
|
||||
"triviaqa_retrieved": "Only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"hotpot_qa": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"squad": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/qasper": 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible.\nIf the question cannot be answered based on the information in the article, write "unanswerable".\nIf the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {input}',
|
||||
"triviaqa_retrieved": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"hotpot_qa": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"squad": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/qasper": 'Answer the question as concisely as you can, using a single phrase or sentence if possible.\nIf the question cannot be answered based on the information in the article, write "unanswerable".\nIf the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {input}',
|
||||
"longbench/narrativeqa": "Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}",
|
||||
"longbench/multifieldqa_en": "Answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/multifieldqa_zh": "阅读以下文字并用中文简短回答：\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}",
|
||||
"longbench/hotpotqa": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/2wikimqa": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/musique": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/dureader": "请基于给定的文章回答下述问题\n\n请基于上述文章回答下面的问题。\n\n问题：{input}",
|
||||
"longbench/multifieldqa_en": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
# "longbench/multifieldqa_zh": "阅读以下文字并用中文简短回答：\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}",
|
||||
"longbench/hotpotqa": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/2wikimqa": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
"longbench/musique": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}",
|
||||
#
|
||||
# "longbench/dureader": "请基于给定的文章回答下述问题\n\n请基于上述文章回答下面的问题。\n\n问题：{input}",
|
||||
"longbench/gov_report": "You are given a report by a government agency. Write a one-page summary of the report.",
|
||||
"longbench/qmsum": "You are given a meeting transcript and a query containing a question or instruction. Answer the query based on the above meeting transcript.\n\nQuery: {input}",
|
||||
"longbench/multi_news": "You are given several news passages. Write a one-page summary of all the news.",
|
||||
"longbench/vcsum": "下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。",
|
||||
# "longbench/vcsum": "下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。",
|
||||
}
|
||||
for ds_name in LONGBENCH_E_TASKS:
|
||||
EVAL_INTX_TEMPLATES[ds_name] = EVAL_INTX_TEMPLATES[ds_name[:-2]]
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ def closed_qa_prompting(prompt: str):
|
|||
return template.format(input=prompt)
|
||||
|
||||
|
||||
# TODO: dont use closed_qa_prompting when training on self-generated responses
|
||||
def get_preprocessing_fn(
|
||||
ds_name: str, is_eval: bool
|
||||
) -> Callable[[dict[str, Any]], dict[str, Any]]:
|
||||
|
|
@ -204,7 +205,6 @@ def get_preprocessing_fn(
|
|||
# ]
|
||||
# )
|
||||
elif ds_name == "triviaqa_retrieved":
|
||||
|
||||
# maybe needed for training (negative sample for training)
|
||||
# def f(sample):
|
||||
# ctx = sample["entity_page"]["wiki_context"]
|
||||
|
|
@ -219,7 +219,6 @@ def get_preprocessing_fn(
|
|||
}
|
||||
|
||||
if is_eval and ds_name in EVAL_INTX_TEMPLATES:
|
||||
|
||||
prompt_template = EVAL_INTX_TEMPLATES[ds_name]
|
||||
|
||||
def eval_intx_decorator(f):
|
||||
|
|
@ -267,9 +266,9 @@ def validate_columns(tokenized_ds):
|
|||
if "chat_ids" in tokenized_ds.column_names:
|
||||
cols += ["chat_ids", "chat_attn_mask", "chat_labels"]
|
||||
ref_cols = set(cols)
|
||||
assert (
|
||||
set(tokenized_ds.column_names) == ref_cols
|
||||
), f"Columns mismatch: {set(tokenized_ds.column_names)} != {ref_cols}"
|
||||
assert set(tokenized_ds.column_names) == ref_cols, (
|
||||
f"Columns mismatch: {set(tokenized_ds.column_names)} != {ref_cols}"
|
||||
)
|
||||
|
||||
|
||||
def filter_long_samples(samples):
|
||||
|
|
@ -297,7 +296,7 @@ def add_repeat_prompt_fn(samples):
|
|||
responses.append(ctx)
|
||||
prompts.append("Repeat the text above.")
|
||||
|
||||
logger.debug(f"Adding repeat prompt...")
|
||||
logger.debug("Adding repeat prompt...")
|
||||
logger.debug(f"# unique contexts: {len(unique_contexts)}")
|
||||
|
||||
return dict(
|
||||
|
|
@ -466,7 +465,7 @@ def get_tokenized_dataset(
|
|||
# TODO: max_len could be included in the kwargs
|
||||
assert not use_kl_loss, "KL loss is deprecated"
|
||||
logger.debug(f"Loading dataset {ds_name} with split {split}...")
|
||||
need_ctx_ids = not add_ctx_to_chat
|
||||
need_ctx_ids = not add_ctx_to_chat and ctx_model_max_len
|
||||
|
||||
load_and_process_kwargs = dict(
|
||||
ds_name=ds_name,
|
||||
|
|
|
|||
|
|
@ -263,13 +263,17 @@ def decode_test_result(test_dataset, test_result, tokenizer, ctx_tokenizer):
|
|||
|
||||
|
||||
@torch.inference_mode()
|
||||
def eval_generation(eval_trainer, tokenizer, ctx_tokenizer, datasets, split, gen_kwargs):
|
||||
def eval_generation(
|
||||
eval_trainer, tokenizer, ctx_tokenizer, datasets, split, remove_context, gen_kwargs
|
||||
):
|
||||
if not isinstance(datasets, dict):
|
||||
datasets = {"": datasets}
|
||||
out = {}
|
||||
for ds_name, ds in datasets.items():
|
||||
print(f"Evaluating: {ds_name}")
|
||||
split_name = f"{split}_{ds_name}" if ds_name else split
|
||||
if remove_context:
|
||||
split_name += "_no_context"
|
||||
eval_result = eval_trainer.predict(
|
||||
ds,
|
||||
metric_key_prefix=split_name,
|
||||
|
|
@ -304,12 +308,14 @@ def eval_generation(eval_trainer, tokenizer, ctx_tokenizer, datasets, split, gen
|
|||
|
||||
|
||||
@torch.no_grad()
|
||||
def eval_teacher_forcing(eval_trainer, datasets, split):
|
||||
def eval_teacher_forcing(eval_trainer, datasets, split, remove_context):
|
||||
if not isinstance(datasets, dict):
|
||||
datasets = {"": datasets}
|
||||
out = {}
|
||||
for ds_name, ds in datasets.items():
|
||||
split_name = f"{split}_{ds_name}" if ds_name else split
|
||||
if remove_context:
|
||||
split_name += "_no_context"
|
||||
metrics = eval_trainer.evaluate(ds, metric_key_prefix=split_name)
|
||||
out[split_name] = metrics
|
||||
eval_trainer.log_metrics(split_name, metrics)
|
||||
|
|
@ -426,7 +432,12 @@ def generation_collator(inp_list, tokenizer):
|
|||
|
||||
|
||||
def evaluate(
|
||||
checkpoint_path, model_name_or_path, eval_batch_size, args, split, generative
|
||||
checkpoint_path,
|
||||
model_name_or_path,
|
||||
eval_batch_size,
|
||||
args,
|
||||
split,
|
||||
generative,
|
||||
):
|
||||
assert split in ["validation", "test"]
|
||||
ctx_name = None
|
||||
|
|
@ -480,7 +491,9 @@ def evaluate(
|
|||
|
||||
tokenizer_kwargs = {}
|
||||
ctx_tokenizer_kwargs = {}
|
||||
add_ctx_to_chat = not isinstance(model, ModulatedPretrainedModel)
|
||||
add_ctx_to_chat = (
|
||||
not isinstance(model, ModulatedPretrainedModel) and not args.remove_context
|
||||
)
|
||||
ctx_model_max_len = (
|
||||
model.ctx_encoder.config.max_position_embeddings
|
||||
if isinstance(model, ModulatedPretrainedModel)
|
||||
|
|
@ -566,12 +579,20 @@ def evaluate(
|
|||
# spents a few hours on this but couldn't find the reason
|
||||
for ds_name, ds in datasets.items():
|
||||
eval_trainer = Trainer(**trainer_kwargs)
|
||||
metrics = eval_teacher_forcing(eval_trainer, {ds_name: ds}, split)
|
||||
metrics = eval_teacher_forcing(
|
||||
eval_trainer, {ds_name: ds}, split, args.remove_context
|
||||
)
|
||||
out.update(metrics)
|
||||
else:
|
||||
eval_trainer = Seq2SeqTrainer(**trainer_kwargs)
|
||||
metrics = eval_generation(
|
||||
eval_trainer, tokenizer, ctx_tokenizer, datasets, split, gen_kwargs
|
||||
eval_trainer,
|
||||
tokenizer,
|
||||
ctx_tokenizer,
|
||||
datasets,
|
||||
split,
|
||||
args.remove_context,
|
||||
gen_kwargs,
|
||||
)
|
||||
out.update(metrics)
|
||||
|
||||
|
|
@ -623,6 +644,11 @@ if __name__ == "__main__":
|
|||
default=32,
|
||||
help="Eval batch size for generation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--remove_context",
|
||||
action="store_true",
|
||||
help="Remove context when evaluating the base model.",
|
||||
)
|
||||
|
||||
cli_args = parser.parse_args()
|
||||
# setup_logging(output_dir, debug=os.getenv("DEBUG", False))
|
||||
|
|
@ -655,6 +681,7 @@ if __name__ == "__main__":
|
|||
args.output_dir = f"{run_dir}/eval-results-{cur_it}"
|
||||
args.logging_dir = f"{run_dir}/eval-results-{cur_it}"
|
||||
args.run_name = run_dir.split("/")[-1]
|
||||
args.remove_context = True
|
||||
else:
|
||||
args = Namespace(
|
||||
model_name_or_path=cli_args.model_name_or_path,
|
||||
|
|
@ -665,6 +692,7 @@ if __name__ == "__main__":
|
|||
# max_ctx_len=-1, # not used
|
||||
val_ds_names=[],
|
||||
test_ds_names=[],
|
||||
remove_context=cli_args.remove_context,
|
||||
)
|
||||
|
||||
# Override dataset names if provided via CLI
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue