From 782a282af4f7d44754e6a542e567859c2bb4936d Mon Sep 17 00:00:00 2001 From: 51616 Date: Fri, 20 Dec 2024 06:48:12 +0000 Subject: [PATCH] compute acc only valid tokens --- hyperlora/intx_sft.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hyperlora/intx_sft.py b/hyperlora/intx_sft.py index a99fd24..b7b1aa5 100644 --- a/hyperlora/intx_sft.py +++ b/hyperlora/intx_sft.py @@ -30,11 +30,12 @@ def compute_metrics(eval_pred) -> dict: # decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True) # compute per token accuracy - predictions, labels = eval_pred.predictions, eval_pred.label_ids + preds, labels = eval_pred.predictions, eval_pred.label_ids - # predictions is logits for Trainer - preds = predictions.argmax(-1) - acc = (preds == labels).mean() + valid_mask = labels != -100 + valid_preds = preds[valid_mask].argmax(-1) + valid_labels = labels[valid_mask] + acc = (valid_preds == valid_labels).mean() return {"per_token_acc": acc}