apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix self-gen val name

Browse source
This commit is contained in:
51616 2025-06-03 16:29:51 +00:00
parent a555a3666b
commit fa809e363b
1 changed files with 8 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

9
intx_sft.py
View file

@ -279,7 +279,14 @@ def main():
streaming = (split == "train") and data_args.streaming
for ds_name in ds_names:
ds = _get_tokenized_dataset(ds_name, split, streaming=streaming)
tokenized_ds[split][os.path.basename(ds_name)] = ds
base_name = os.path.basename(ds_name)
if ds_name.startswith("self_gen/"):
ds_name = "self_gen/" + base_name
else:
ds_name = base_name
tokenized_ds[split][ds_name] = ds
train_ds = tokenized_ds["train"]
logging.info(f"train_ds: {train_ds}")