default warmup + decay

2026-07-23 17:01:04 +02:00 · 2025-01-28 21:36:19 +00:00 · 2025-01-28 21:36:19 +00:00 · 4bad251877
commit 4bad251877
parent a447039858
27 changed files with 44 additions and 38 deletions
--- a/configs/context_numbers_10.yaml
+++ b/configs/context_numbers_10.yaml
@ -27,8 +27,7 @@ max_val_samples_per_ds: 500
 learning_rate: 0.0001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_128.yaml
+++ b/configs/context_numbers_128.yaml
@ -26,8 +26,8 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_128_only.yaml
+++ b/configs/context_numbers_128_only.yaml
@ -26,8 +26,8 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_256.yaml
+++ b/configs/context_numbers_256.yaml
@ -26,8 +26,8 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_256_only.yaml
+++ b/configs/context_numbers_256_only.yaml
@ -26,8 +26,8 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_32.yaml
+++ b/configs/context_numbers_32.yaml
@ -26,8 +26,7 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_32_only.yaml
+++ b/configs/context_numbers_32_only.yaml
@ -26,8 +26,7 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_64.yaml
+++ b/configs/context_numbers_64.yaml
@ -26,8 +26,8 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_64_only.yaml
+++ b/configs/context_numbers_64_only.yaml
@ -26,8 +26,8 @@ max_val_samples_per_ds: 20
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.05
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_debug.yaml
+++ b/configs/context_numbers_debug.yaml
@ -26,7 +26,7 @@ learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 # LoRA
 lora_r: 16
--- a/configs/context_numbers_easy.yaml
+++ b/configs/context_numbers_easy.yaml
@ -26,7 +26,7 @@ learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 # LoRA
 lora_r: 16
--- a/configs/default.yaml
+++ b/configs/default.yaml
@ -24,7 +24,7 @@ learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 # LoRA
 lora_r: 8
--- a/configs/fw_qa.yaml
+++ b/configs/fw_qa.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 dataloader_prefetch_factor: 8
 dataloader_num_workers: 8
--- a/configs/fw_qa_ctx_qa.yaml
+++ b/configs/fw_qa_ctx_qa.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 dataloader_prefetch_factor: 8
 dataloader_num_workers: 8
--- a/configs/fw_qa_large_ctx_pwc_hotpot.yaml
+++ b/configs/fw_qa_large_ctx_pwc_hotpot.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 dataloader_prefetch_factor: 8
 dataloader_num_workers: 8
--- a/configs/fw_qa_large_ctx_pwc_hotpot_squad.yaml
+++ b/configs/fw_qa_large_ctx_pwc_hotpot_squad.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-# warmup_ratio: 0.1
+# 
 warmup_steps: 100

 dataloader_prefetch_factor: 8
--- a/configs/fw_qa_pwc_hotpot.yaml
+++ b/configs/fw_qa_pwc_hotpot.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 dataloader_prefetch_factor: 8
 dataloader_num_workers: 8
--- a/configs/fw_qa_tiny.yaml
+++ b/configs/fw_qa_tiny.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 dataloader_prefetch_factor: 8
 dataloader_num_workers: 8
--- a/configs/gsm8k.yaml
+++ b/configs/gsm8k.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-# warmup_ratio: 0.1
+# 
 warmup_steps: 100

 dataloader_prefetch_factor: 8
--- a/configs/hotpot_qa.yaml
+++ b/configs/hotpot_qa.yaml
@ -23,8 +23,8 @@ max_val_samples_per_ds: 1000
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.1
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/math_and_code.yaml
+++ b/configs/math_and_code.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-# warmup_ratio: 0.1
+# 
 warmup_steps: 100

 dataloader_prefetch_factor: 8
--- a/configs/openmathintx.yaml
+++ b/configs/openmathintx.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-# warmup_ratio: 0.1
+# 
 warmup_steps: 100

 dataloader_prefetch_factor: 8
--- a/configs/pretrain_all.yaml
+++ b/configs/pretrain_all.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 5
 weight_decay: 0.01
-# warmup_ratio: 0.1
+# 
 warmup_steps: 100

 dataloader_prefetch_factor: 8
--- a/configs/pwc.yaml
+++ b/configs/pwc.yaml
@ -23,8 +23,8 @@ max_val_samples_per_ds: 1000
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.1
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/pwc_and_ctx_numbers_256.yaml
+++ b/configs/pwc_and_ctx_numbers_256.yaml
@ -23,8 +23,8 @@ max_val_samples_per_ds: 1000
 learning_rate: 0.00001
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
-weight_decay: 0.1
-warmup_ratio: 0.1
+weight_decay: 0.01
+

 # LoRA
 lora_r: 16
--- a/configs/pwc_hotpot_qa.yaml
+++ b/configs/pwc_hotpot_qa.yaml
@ -25,7 +25,7 @@ learning_rate: 0.00002
 # lr_scheduler_type: "constant_with_warmup"
 neftune_noise_alpha: 1
 weight_decay: 0.01
-warmup_ratio: 0.1
+

 # LoRA
 lora_r: 8
--- a/src/ctx_to_lora/configs.py
+++ b/src/ctx_to_lora/configs.py
@ -49,6 +49,11 @@ class ArgumentParser(HfArgumentParser):
            for arg, val in other_args.items():
                # add only if in keys
                if arg in keys:
+                    if val in ["None", "none", "null", "NULL"]:
+                        val = None
+                        inputs[arg] = val
+                        used_args[arg] = val
+                        continue
                    base_type = data_yaml.__dataclass_fields__[arg].type
                    inputs[arg] = val

@ -166,6 +171,10 @@ class TrainingArguments(TrainingArguments):
        default=None,
        metadata={"help": "Learning rate scheduler kwargs."},
    )
+    warmup_steps: int = field(
+        default=100,
+        metadata={"help": "Number of warmup steps."},
+    )
    eval_on_start: bool = field(
        default=True,
        metadata={"help": "Whether to evaluate on the start of training."},