From d6c7e3ab03afd356f9ba4ed9dbf16acd0c70d23d Mon Sep 17 00:00:00 2001 From: 51616 Date: Mon, 10 Nov 2025 16:54:25 +0900 Subject: [PATCH] fix tracker + recall/precision logging + add generative adapter --- run_eval.py | 5 +++++ src/ctx_to_lora/eval_utils.py | 25 ++++++++++++++++++++----- src/ctx_to_lora/tracker/tracker.py | 2 +- 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/run_eval.py b/run_eval.py index 5796609..44f52c9 100644 --- a/run_eval.py +++ b/run_eval.py @@ -152,6 +152,11 @@ if __name__ == "__main__": action="store_true", help="Flip the order of context and input", ) + parser.add_argument( + "--use_generative_adapter", + action="store_true", + help="Use generative adapter for evaluation", + ) cli_args = vars(parser.parse_args()) diff --git a/src/ctx_to_lora/eval_utils.py b/src/ctx_to_lora/eval_utils.py index 60b4198..c848b0e 100644 --- a/src/ctx_to_lora/eval_utils.py +++ b/src/ctx_to_lora/eval_utils.py @@ -52,6 +52,7 @@ from ctx_to_lora.model_loading import ( get_tokenizer, ) from ctx_to_lora.modeling.context_distillation import CtxDistillModel +from ctx_to_lora.modeling.generative_adapter import GenerativeAdapter from ctx_to_lora.modeling.hypernet import ModulatedPretrainedModel from ctx_to_lora.modeling.llm_lingua import LLMLinguaModel from ctx_to_lora.modeling.text_to_lora import TextToLoRA @@ -121,7 +122,9 @@ def split_string(s: str) -> list[str]: return [x for x in out if x] # remove empty spaces -def f1_score(prediction: str, ground_truth: str) -> tuple[float, float, float]: +def f1_score( + prediction: list[str], ground_truth: list[str] +) -> tuple[float, float, float]: """Compute F1 score, precision, and recall between prediction and ground truth strings.""" common = Counter(prediction) & Counter(ground_truth) num_same = sum(common.values()) @@ -200,11 +203,11 @@ def save_generated_text( metric_keys = list(per_sample_metric.keys()) # assert len(metric_keys) == 1 - metric_name = metric_keys[0] + with open(f"{output_dir}/{split}_generated_text.jsonl", "w") as f: - for sample, metric_val in zip(samples, per_sample_metric[metric_name]): + for i, sample in enumerate(samples): for metric_name in metric_keys: - sample[f"{metric_name}"] = metric_val + sample[f"{metric_name}"] = per_sample_metric[metric_name][i] f.write(json.dumps(sample) + "\n") @@ -771,7 +774,7 @@ def evaluate( add_special_tokens=False, return_tensors="pt", ) - .input_ids[0] + .input_ids[0][1:] .to(base_model.device) ) ctx_distill_kwargs = dict( @@ -819,6 +822,10 @@ def evaluate( add_tracker(model.base_model.generate, "base_model.generate") add_tracker(model.generate_weights, "generate_weights") + elif args.use_generative_adapter: + model = GenerativeAdapter(model=base_model, tokenizer=tokenizer) + ctx_model_max_len = base_model.config.max_position_embeddings + if base_model is None: base_model = model.base_model base_model.config.pad_token_id = tokenizer.pad_token_id @@ -1027,6 +1034,7 @@ def run_eval( truncate_if_too_long_ctx: bool = False, flip_ctx_inp: bool = False, gen_lora_scaling: float = 1, + use_generative_adapter: bool = False, ) -> None: """Run evaluation with the specified parameters.""" assert bool(model_name_or_path) ^ bool(checkpoint_path), ( @@ -1039,6 +1047,12 @@ def run_eval( raise ValueError( "LLMLingua always adds compressed context to input by default." ) + if use_generative_adapter and ( + model_name_or_path != "mistralai/Mistral-7B-Instruct-v0.2" + ): + raise ValueError( + "Generative adapter is only available for Mistral-7B-Instruct-v0.2." + ) disable_caching() set_seed(42) @@ -1099,6 +1113,7 @@ def run_eval( args.llmlingua_compression_rate = llmlingua_compression_rate if use_t2l: args.use_t2l = use_t2l + args.use_generative_adapter = use_generative_adapter if max_val_samples_per_ds > 0: args.max_val_samples_per_ds = max_val_samples_per_ds if max_test_samples_per_ds > 0: diff --git a/src/ctx_to_lora/tracker/tracker.py b/src/ctx_to_lora/tracker/tracker.py index 6077f32..838d004 100644 --- a/src/ctx_to_lora/tracker/tracker.py +++ b/src/ctx_to_lora/tracker/tracker.py @@ -224,7 +224,7 @@ def save_tracker_stats_csv(file_path: str, name: str | None = None) -> None: # If no data at all, raise an error if timer_stats is None and memory_stats is None: - raise ValueError("No tracking data available to export") + print("No tracking data available to export") def print_tracker_stats(name: str | None = None) -> None: