This commit is contained in:
51616 2025-01-04 18:24:33 +00:00
parent b9c9dc3615
commit 9d22b804b7

View file

@ -391,7 +391,7 @@ def main(output_dir):
if "validation" in tokenized_ds:
val_ds["validation"] = tokenized_ds["validation"]
val_ds_size = len(val_ds["validation"])
val_indices = np.random.permutation(val_ds_size)[: args.max_val_samples]
val_indices = np.random.permutation(val_ds_size)[: data_args.max_val_samples]
val_ds["validation"] = val_ds["validation"].select(val_indices)
test_ds = tokenized_ds.get("test", None)