apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

context distillation eval + timer

Browse source
This commit is contained in:
51616 2025-09-12 16:34:03 +09:00
parent f2efe22cba
commit 7d95db6672
17 changed files with 1840 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

5
train.py
View file

@ -302,6 +302,11 @@ def main():
tokenized_ds[split][ds_name] = ds
train_ds = tokenized_ds["train"]
if data_args.max_train_samples_per_ds is not None:
for ds_name, ds in train_ds.items():
if data_args.max_train_samples_per_ds >= len(ds):
continue
train_ds[ds_name] = ds.take(data_args.max_train_samples_per_ds)
logging.info(f"train_ds: {train_ds}")
val_ds = dict()