From 83ffaefd9bf6ade18a34fcbc40dcbb02914936e7 Mon Sep 17 00:00:00 2001 From: 51616 Date: Thu, 12 Jun 2025 07:39:48 +0000 Subject: [PATCH] fix eval (logging + reformat csv) --- src/ctx_to_lora/eval_utils.py | 126 +++++++++++++++++++++++++--------- src/ctx_to_lora/metrics.py | 27 ++++---- 2 files changed, 105 insertions(+), 48 deletions(-) diff --git a/src/ctx_to_lora/eval_utils.py b/src/ctx_to_lora/eval_utils.py index f2a17f8..57c873c 100644 --- a/src/ctx_to_lora/eval_utils.py +++ b/src/ctx_to_lora/eval_utils.py @@ -161,6 +161,9 @@ def _extract_model_info(eval_trainer) -> tuple[str, bool]: if "/" in model_name: model_name = model_name.split("/")[-1] + if getattr(eval_trainer.args, "run_name", None): + model_name += f"_{eval_trainer.args.run_name}" + return model_name, is_hypernet @@ -215,7 +218,7 @@ def _sort_length_groups(length_groups: set[str]) -> list[str]: return (1, 0, 0) # Put "overall" after numerical ranges try: # Parse "low-high" format - low, high = map(int, length_group.split("-")) + low, high = map(float, length_group.split("-")) return (0, low, high) except (ValueError, IndexError): return (2, 0, 0) # Put any malformed strings last @@ -232,8 +235,8 @@ def create_metrics_csv( csv_suffix: str = "", ) -> None: """ - Create a human-readable CSV file from evaluation metrics with hierarchical columns. - One row per model with columns structured as task_metric_lengthgroup. + Create a CSV file with columns: model_name, group_len, tasks, num_samples, and all available metrics. + One row per model-length-task combination. Args: metrics_dict: Dictionary containing evaluation metrics for each dataset/split @@ -249,48 +252,94 @@ def create_metrics_csv( all_metrics, all_length_groups, all_splits = _parse_metrics_for_csv(metrics_dict) # Sort for consistent ordering - all_metrics = sorted(all_metrics) all_length_groups = _sort_length_groups(all_length_groups) all_splits = sorted(all_splits) - # Create single row with hierarchical columns - row_data = { - "model": model_name, - "model_type": "hypernet" if is_hypernet_model else "base", - } + # Create rows for each model-length-task combination + rows = [] - # Create hierarchical column structure: task_metric_lengthgroup for task in all_splits: metrics = metrics_dict[task] - is_qa_task = any(qa_task in task for qa_task in CLOSED_QA_DATASETS) - for metric in all_metrics: - # Skip inappropriate metrics for the task type - if is_qa_task and ("rougeL" in metric): - continue - if not is_qa_task and ("qa_f1" in metric): - continue + for length_group in all_length_groups: + # Initialize row with basic info + row = { + "model_name": model_name, + "group_len": length_group, + "tasks": task, + "num_samples": 0, + } - for length_group in all_length_groups: - if length_group == "overall": - metric_key = f"{task}_{metric}" - column_name = f"{task}_{metric}_overall" - else: - metric_key = f"{task}_{metric}_len_{length_group}" - column_name = f"{task}_{metric}_{length_group}" + # Look for all metrics for this task and length group + if length_group == "overall": + # Look for overall metrics (no length suffix) + for metric_key in metrics: + if not metric_key.startswith(f"{task}_"): + continue - value = metrics.get(metric_key, "N/A") - # Handle None values and convert to string - if value is None or value == "None": - value = "N/A" - elif isinstance(value, (int, float)): - value = f"{value:.4f}" + # Skip length-specific metrics + if "_len_" in metric_key: + continue - row_data[column_name] = value + # Extract metric name after task prefix + metric_name = metric_key[len(task) + 1 :] + if metric_name in [ + "samples_per_second", + "steps_per_second", + "model_preparation_time", + "runtime", + ]: + # Skip timing and performance metrics + continue - # Create DataFrame with single row - if len(row_data) > 2: # More than just model and model_type - df = pd.DataFrame([row_data]) + # Handle num_samples specially + if metric_name.startswith("num_samples_"): + row["num_samples"] = metrics[metric_key] + else: + # Add all other metrics as columns + row[metric_name] = metrics[metric_key] + else: + # Look for length-specific metrics + for metric_key in metrics: + if not metric_key.startswith(f"{task}_"): + continue + + # Only process metrics for this specific length group + if f"_len_{length_group}" not in metric_key: + continue + + # Extract metric name (remove task prefix and length suffix) + metric_part = metric_key[len(task) + 1 :] + metric_name = metric_part.replace(f"_len_{length_group}", "") + if metric_name in [ + "samples_per_second", + "steps_per_second", + "model_preparation_time", + "runtime", + ]: + # Skip timing and performance metrics + continue + + # Handle num_samples specially + if metric_name.startswith("num_samples_"): + row["num_samples"] = metrics[metric_key] + else: + # Add all other metrics as columns + row[metric_name] = metrics[metric_key] + + # Fill missing metrics with N/A for consistent columns + for metric in all_metrics: + if metric not in row and not metric.startswith("num_samples"): + row[metric] = "N/A" + + rows.append(row) + + # Create DataFrame + if rows: + df = pd.DataFrame(rows) + + # Sort by tasks, then by length group + df = df.sort_values(["tasks"]).reset_index(drop=True) # Construct filename csv_filename = "evaluation_results" @@ -594,6 +643,15 @@ def evaluate( datasets[ds_name] = _get_tokenized_dataset(ds_name, split) print(datasets) + # truncating num val samples + max_eval_samples_per_ds = getattr(args, "max_val_samples_per_ds", 0) + if split == "validation" and max_eval_samples_per_ds > 0: + print(f"Truncating all validation ds to {max_eval_samples_per_ds} samples") + for ds_name, ds in datasets.items(): + val_indices = np.random.permutation(len(ds))[:max_eval_samples_per_ds] + datasets[ds_name] = ds.select(val_indices) + print(datasets) + gen_kwargs = dict( do_sample=False, max_new_tokens=256, diff --git a/src/ctx_to_lora/metrics.py b/src/ctx_to_lora/metrics.py index c44665b..56c0409 100644 --- a/src/ctx_to_lora/metrics.py +++ b/src/ctx_to_lora/metrics.py @@ -100,12 +100,11 @@ class Evaluator: else: self.accum_metrics[key] += v for start, end in LENGTH_BINS: - key = f"{k}_len_{start}_{end}" - if key not in self.accum_metrics: + key_w_len = f"{key}_len_{start}-{end}" + if key_w_len not in self.accum_metrics: # add key here so that it shows up in the output - self.accum_metrics[key] = [0] - self.count[key] = [0] - + self.accum_metrics[key_w_len] = [0] + self.count[key_w_len] = [0] # split samples into length groups, calculate metric for each group if lengths is not None: for start, end in LENGTH_BINS: @@ -129,11 +128,11 @@ class Evaluator: ) for k, v in metric.items(): if k.startswith("n_"): - key = f"{k[2:]}_len_{start}_{end}" - self.count[f"{k[2:]}_len_{start}_{end}"].append(v) + key = f"{k[2:]}_len_{start}-{end}" + self.count[key].append(v) else: - key = f"{k}_len_{start}_{end}" - self.accum_metrics[f"{k}_len_{start}_{end}"] += v + key = f"{k}_len_{start}-{end}" + self.accum_metrics[key] += v def compute(self): # Get result across entire eval set @@ -141,11 +140,11 @@ class Evaluator: k: np.sum(v) / np.sum(self.count[k]) if np.sum(v) != 0 else "None" for k, v in self.accum_metrics.items() } - for k, v in self.count.items(): - if "_len_" in k: - result[k.replace("_len_", "_num_samples_len_")] = sum(v) - else: - result[k + "_num_samples"] = sum(v) + # for k, v in self.count.items(): + # if "_len_" in k: + # result[k.replace("_len_", "_num_samples_len_")] = sum(v) + # else: + # result[k + "_num_samples"] = sum(v) # Reset batch statistics self.reset() return result