From 19ca0f99a59868f7e2a05a25ff5c419ff9f62e31 Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 20 May 2025 09:43:08 +0000 Subject: [PATCH] eval prompts + no context --- src/ctx_to_lora/data_definitions.py | 23 +++++++++-------- src/ctx_to_lora/data_utils.py | 13 +++++----- src/ctx_to_lora/eval.py | 40 ++++++++++++++++++++++++----- 3 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/ctx_to_lora/data_definitions.py b/src/ctx_to_lora/data_definitions.py index ebfb1b7..40d67fc 100644 --- a/src/ctx_to_lora/data_definitions.py +++ b/src/ctx_to_lora/data_definitions.py @@ -341,21 +341,22 @@ CLOSED_QA_INTX_TEMPLATES = [ EVAL_INTX_TEMPLATES = { - "triviaqa_retrieved": "Only give me the answer and do not output any other words.\n\nQuestion: {input}", - "hotpot_qa": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}", - "squad": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}", - "longbench/qasper": 'You are given a scientific article and a question. Answer the question as concisely as you can, using a single phrase or sentence if possible.\nIf the question cannot be answered based on the information in the article, write "unanswerable".\nIf the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {input}', + "triviaqa_retrieved": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", + "hotpot_qa": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", + "squad": "Answer the following question. Output only the answer and do not output any other words.\n\nQuestion: {input}", + "longbench/qasper": 'Answer the question as concisely as you can, using a single phrase or sentence if possible.\nIf the question cannot be answered based on the information in the article, write "unanswerable".\nIf the question is a yes/no question, answer "yes", "no", or "unanswerable". Do not provide any explanation.\n\nQuestion: {input}', "longbench/narrativeqa": "Answer the question as concisely as you can, using a single phrase if possible. Do not provide any explanation.\n\nQuestion: {input}", - "longbench/multifieldqa_en": "Answer the following question based on the above text, only give me the answer and do not output any other words.\n\nQuestion: {input}", - "longbench/multifieldqa_zh": "阅读以下文字并用中文简短回答：\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}", - "longbench/hotpotqa": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}", - "longbench/2wikimqa": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}", - "longbench/musique": "Answer the question based on the given passages. Only give me the answer and do not output any other words.\n\nQuestion: {input}", - "longbench/dureader": "请基于给定的文章回答下述问题\n\n请基于上述文章回答下面的问题。\n\n问题：{input}", + "longbench/multifieldqa_en": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", + # "longbench/multifieldqa_zh": "阅读以下文字并用中文简短回答：\n\n现在请基于上面的文章回答下面的问题，只告诉我答案，不要输出任何其他字词。\n\n问题：{input}", + "longbench/hotpotqa": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", + "longbench/2wikimqa": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", + "longbench/musique": "Answer the following question. Only output the answer and do not output any other words.\n\nQuestion: {input}", + # + # "longbench/dureader": "请基于给定的文章回答下述问题\n\n请基于上述文章回答下面的问题。\n\n问题：{input}", "longbench/gov_report": "You are given a report by a government agency. Write a one-page summary of the report.", "longbench/qmsum": "You are given a meeting transcript and a query containing a question or instruction. Answer the query based on the above meeting transcript.\n\nQuery: {input}", "longbench/multi_news": "You are given several news passages. Write a one-page summary of all the news.", - "longbench/vcsum": "下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。", + # "longbench/vcsum": "下面有一段会议记录，请你阅读后，写一段总结，总结会议的内容。", } for ds_name in LONGBENCH_E_TASKS: EVAL_INTX_TEMPLATES[ds_name] = EVAL_INTX_TEMPLATES[ds_name[:-2]] diff --git a/src/ctx_to_lora/data_utils.py b/src/ctx_to_lora/data_utils.py index 6492b2d..2806fa2 100644 --- a/src/ctx_to_lora/data_utils.py +++ b/src/ctx_to_lora/data_utils.py @@ -32,6 +32,7 @@ def closed_qa_prompting(prompt: str): return template.format(input=prompt) +# TODO: dont use closed_qa_prompting when training on self-generated responses def get_preprocessing_fn( ds_name: str, is_eval: bool ) -> Callable[[dict[str, Any]], dict[str, Any]]: @@ -204,7 +205,6 @@ def get_preprocessing_fn( # ] # ) elif ds_name == "triviaqa_retrieved": - # maybe needed for training (negative sample for training) # def f(sample): # ctx = sample["entity_page"]["wiki_context"] @@ -219,7 +219,6 @@ def get_preprocessing_fn( } if is_eval and ds_name in EVAL_INTX_TEMPLATES: - prompt_template = EVAL_INTX_TEMPLATES[ds_name] def eval_intx_decorator(f): @@ -267,9 +266,9 @@ def validate_columns(tokenized_ds): if "chat_ids" in tokenized_ds.column_names: cols += ["chat_ids", "chat_attn_mask", "chat_labels"] ref_cols = set(cols) - assert ( - set(tokenized_ds.column_names) == ref_cols - ), f"Columns mismatch: {set(tokenized_ds.column_names)} != {ref_cols}" + assert set(tokenized_ds.column_names) == ref_cols, ( + f"Columns mismatch: {set(tokenized_ds.column_names)} != {ref_cols}" + ) def filter_long_samples(samples): @@ -297,7 +296,7 @@ def add_repeat_prompt_fn(samples): responses.append(ctx) prompts.append("Repeat the text above.") - logger.debug(f"Adding repeat prompt...") + logger.debug("Adding repeat prompt...") logger.debug(f"# unique contexts: {len(unique_contexts)}") return dict( @@ -466,7 +465,7 @@ def get_tokenized_dataset( # TODO: max_len could be included in the kwargs assert not use_kl_loss, "KL loss is deprecated" logger.debug(f"Loading dataset {ds_name} with split {split}...") - need_ctx_ids = not add_ctx_to_chat + need_ctx_ids = not add_ctx_to_chat and ctx_model_max_len load_and_process_kwargs = dict( ds_name=ds_name, diff --git a/src/ctx_to_lora/eval.py b/src/ctx_to_lora/eval.py index b7486d5..50bb7e0 100644 --- a/src/ctx_to_lora/eval.py +++ b/src/ctx_to_lora/eval.py @@ -263,13 +263,17 @@ def decode_test_result(test_dataset, test_result, tokenizer, ctx_tokenizer): @torch.inference_mode() -def eval_generation(eval_trainer, tokenizer, ctx_tokenizer, datasets, split, gen_kwargs): +def eval_generation( + eval_trainer, tokenizer, ctx_tokenizer, datasets, split, remove_context, gen_kwargs +): if not isinstance(datasets, dict): datasets = {"": datasets} out = {} for ds_name, ds in datasets.items(): print(f"Evaluating: {ds_name}") split_name = f"{split}_{ds_name}" if ds_name else split + if remove_context: + split_name += "_no_context" eval_result = eval_trainer.predict( ds, metric_key_prefix=split_name, @@ -304,12 +308,14 @@ def eval_generation(eval_trainer, tokenizer, ctx_tokenizer, datasets, split, gen @torch.no_grad() -def eval_teacher_forcing(eval_trainer, datasets, split): +def eval_teacher_forcing(eval_trainer, datasets, split, remove_context): if not isinstance(datasets, dict): datasets = {"": datasets} out = {} for ds_name, ds in datasets.items(): split_name = f"{split}_{ds_name}" if ds_name else split + if remove_context: + split_name += "_no_context" metrics = eval_trainer.evaluate(ds, metric_key_prefix=split_name) out[split_name] = metrics eval_trainer.log_metrics(split_name, metrics) @@ -426,7 +432,12 @@ def generation_collator(inp_list, tokenizer): def evaluate( - checkpoint_path, model_name_or_path, eval_batch_size, args, split, generative + checkpoint_path, + model_name_or_path, + eval_batch_size, + args, + split, + generative, ): assert split in ["validation", "test"] ctx_name = None @@ -480,7 +491,9 @@ def evaluate( tokenizer_kwargs = {} ctx_tokenizer_kwargs = {} - add_ctx_to_chat = not isinstance(model, ModulatedPretrainedModel) + add_ctx_to_chat = ( + not isinstance(model, ModulatedPretrainedModel) and not args.remove_context + ) ctx_model_max_len = ( model.ctx_encoder.config.max_position_embeddings if isinstance(model, ModulatedPretrainedModel) @@ -566,12 +579,20 @@ def evaluate( # spents a few hours on this but couldn't find the reason for ds_name, ds in datasets.items(): eval_trainer = Trainer(**trainer_kwargs) - metrics = eval_teacher_forcing(eval_trainer, {ds_name: ds}, split) + metrics = eval_teacher_forcing( + eval_trainer, {ds_name: ds}, split, args.remove_context + ) out.update(metrics) else: eval_trainer = Seq2SeqTrainer(**trainer_kwargs) metrics = eval_generation( - eval_trainer, tokenizer, ctx_tokenizer, datasets, split, gen_kwargs + eval_trainer, + tokenizer, + ctx_tokenizer, + datasets, + split, + args.remove_context, + gen_kwargs, ) out.update(metrics) @@ -623,6 +644,11 @@ if __name__ == "__main__": default=32, help="Eval batch size for generation", ) + parser.add_argument( + "--remove_context", + action="store_true", + help="Remove context when evaluating the base model.", + ) cli_args = parser.parse_args() # setup_logging(output_dir, debug=os.getenv("DEBUG", False)) @@ -655,6 +681,7 @@ if __name__ == "__main__": args.output_dir = f"{run_dir}/eval-results-{cur_it}" args.logging_dir = f"{run_dir}/eval-results-{cur_it}" args.run_name = run_dir.split("/")[-1] + args.remove_context = True else: args = Namespace( model_name_or_path=cli_args.model_name_or_path, @@ -665,6 +692,7 @@ if __name__ == "__main__": # max_ctx_len=-1, # not used val_ds_names=[], test_ds_names=[], + remove_context=cli_args.remove_context, ) # Override dataset names if provided via CLI