From fa809e363bc952c039a6c58b569624baffd6c0f1 Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 3 Jun 2025 16:29:51 +0000 Subject: [PATCH] fix self-gen val name --- intx_sft.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/intx_sft.py b/intx_sft.py index 84fa251..0b191ba 100755 --- a/intx_sft.py +++ b/intx_sft.py @@ -279,7 +279,14 @@ def main(): streaming = (split == "train") and data_args.streaming for ds_name in ds_names: ds = _get_tokenized_dataset(ds_name, split, streaming=streaming) - tokenized_ds[split][os.path.basename(ds_name)] = ds + + base_name = os.path.basename(ds_name) + if ds_name.startswith("self_gen/"): + ds_name = "self_gen/" + base_name + else: + ds_name = base_name + + tokenized_ds[split][ds_name] = ds train_ds = tokenized_ds["train"] logging.info(f"train_ds: {train_ds}")