apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

repeat_prompts + faster eval

Browse source
This commit is contained in:
51616 2025-07-29 06:59:58 +00:00
parent d99fa8732b
commit 29719f4d19
3 changed files with 33 additions and 27 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/ctx_to_lora/configs.py
View file

@ -144,6 +144,10 @@ class TrainingArguments(TrainingArguments):
"Options: 'inputs', 'loss'."
},
)
average_tokens_across_devices: bool = field(
default=False,
metadata={"help": "compute num_items_in_batch across devices."},
)
# mem leak if use persistent workers
# https://github.com/pytorch/pytorch/issues/62066
# https://github.com/huggingface/transformers/issues/30943

47
src/ctx_to_lora/data/definitions.py
View file

@ -22,30 +22,29 @@ SELF_GEN_SYSTEM_MSG = (
REPEAT_PROMPTS = [
"Repeat the text.",
"Repeat the text above.",
"Repeat the above text.",
"Repeat exactly what was written.",
"Duplicate the text.",
"Echo the text. Do not output other words.",
"Repeat the content provided.",
"Copy the content. Reply with just the content.",
"Repeat what you see above. Avoid explanation.",
"Replicate the given text precisely.",
"Repeat the entire passage.",
"Duplicate exactly the provided text.",
"Repeat the text that appears above.",
"Reproduce exactly what appears above.",
"Repeat the article.",
"Repeat verbatim the given text.",
"Output the same text as shown.",
"Output an exact copy of all of the content.",
"Provide a replica of the text.",
"Output an identical copy of the text.",
"Repeat the content without any changes.",
"Repeat the provided text.",
"Repeat the provided text above.",
"Repeat precisely what is written in the text above.",
"Repeat the information.",
"Repeat the information above.",
"Repeat the above information.",
# "Repeat exactly what was written.",
"Duplicate the information.",
"Echo the information. Do not output other words.",
"Repeat the information provided.",
"Copy the information. Reply with just the information.",
"Replicate the given information precisely.",
"Repeat the entire provided information above.",
"Duplicate exactly the provided information.",
"Repeat the information that appears above.",
# "Reproduce exactly what appears above.",
# "Repeat the article.",
"Repeat verbatim the given information.",
"Output the same information as provided.",
"Output an exact copy of the information above.",
"Provide a replica of the information.",
"Output an identical copy of the information.",
"Repeat the information without any changes.",
"Repeat the provided information.",
"Repeat the provided information above.",
"Repeat precisely what is written in the information above.",
]
# approximate length of the datasets (train split)

9
src/ctx_to_lora/eval_utils.py
View file

@ -798,9 +798,11 @@ def evaluate(
eval_trainer_args["include_for_metrics"] = ["inputs"]
eval_trainer_args["batch_eval_metrics"] = True
eval_trainer_args["remove_unused_columns"] = False
eval_trainer_args["bf16"] = True
eval_trainer_args["tf32"] = True
eval_trainer_args["use_liger_kernel"] = True
eval_trainer_args["bf16"] = False
eval_trainer_args["tf32"] = False
eval_trainer_args["use_liger_kernel"] = False
eval_trainer_args["dataloader_num_workers"] = 0
eval_trainer_args["dataloader_prefetch_factor"] = None
eval_trainer_args = Seq2SeqTrainingArguments(
**eval_trainer_args,
@ -877,6 +879,7 @@ def run_eval(
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
os.environ["TRANSFORMERS_NO_ADVISORY_WARNINGS"] = "true"
os.environ["FLASH_ATTENTION_DETERMINISTIC"] = "1"
os.environ["OMP_NUM_THREADS"] = "23"
# torch.use_deterministic_algorithms(True, warn_only=True)
torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False