apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

clean up + configs + scripts

Browse source
This commit is contained in:
51616 2025-06-24 21:51:33 +09:00
parent 1ec757f43a
commit 816377b4ac
37 changed files with 216 additions and 1091 deletions
Download patch file Download diff file Expand all files Collapse all files

9
README.md
View file

@ -33,9 +33,14 @@ print(model.decode(outputs))
```bash
DEBUG=1 WANDB_MODE=disabled uv run intx_sft.py configs/context_numbers_10.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=1 --per_device_train_batch_size=64 --gradient_accumulation_steps=1 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=2 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=False --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --use_sequence_packing=True --max_packed_inp_len=8000 --max_packed_ctx_len=16000 --dataloader_num_workers=0 --dataloader_prefetch_factor=None --eval_on_start=True
```
### Squad + Hotpot training (for testing/debugging)
### FWQA-v2 Level-0 Tiny
```bash
WANDB_MODE=disabled uv run intx_sft.py configs/squad.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --metric_for_best_model=eval_pwc_loss --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=16000 --max_packed_ctx_len=32000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=10
WANDB_MODE=disabled run uv run intx_sft.py configs/fw_qa_v2_level_0_tiny.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=24000 --max_packed_ctx_len=48000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=50
```
### Squad only
```bash
WANDB_MODE=disabled uv run intx_sft.py configs/squad.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=16000 --max_packed_ctx_len=32000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=10
```
### HyperLoRA w/ self-gen 3 mini

65
configs/data_exp/qa.yaml
View file

@ -1,65 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small # ~ 20M
- ctx_qa # 300k
- pwc # 240k
- hotpot_qa # 90k
- squad # 90k
- drop # 77k
- narrativeqa # 40k
- quoref # 11k
- ropes # 11k
- synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

66
configs/data_exp/self_gen_3_and_pretrain_small.yaml
View file

@ -1,66 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small_pretrain # ~6M
- self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M
- self_gen/google/gemma-2-2b-it/ctx_qa # 300k
- self_gen/google/gemma-2-2b-it/pwc # 240k
- self_gen/google/gemma-2-2b-it/hotpot_qa # 90k
- self_gen/google/gemma-2-2b-it/squad # 90k
- self_gen/google/gemma-2-2b-it/drop # 77k
- self_gen/google/gemma-2-2b-it/narrativeqa # 40k
- self_gen/google/gemma-2-2b-it/quoref # 11k
- self_gen/google/gemma-2-2b-it/ropes # 11k
- self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

67
configs/data_exp/self_gen_3_and_pretrain_small_aug.yaml
View file

@ -1,67 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small_pretrain # ~6M
- fw_qa_3_small_aug_pretrain # ~12M
- self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M
- self_gen/google/gemma-2-2b-it/ctx_qa # 300k
- self_gen/google/gemma-2-2b-it/pwc # 240k
- self_gen/google/gemma-2-2b-it/hotpot_qa # 90k
- self_gen/google/gemma-2-2b-it/squad # 90k
- self_gen/google/gemma-2-2b-it/drop # 77k
- self_gen/google/gemma-2-2b-it/narrativeqa # 40k
- self_gen/google/gemma-2-2b-it/quoref # 11k
- self_gen/google/gemma-2-2b-it/ropes # 11k
- self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

65
configs/data_exp/self_gen_3_small.yaml
View file

@ -1,65 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M
- self_gen/google/gemma-2-2b-it/ctx_qa # 300k
- self_gen/google/gemma-2-2b-it/pwc # 240k
- self_gen/google/gemma-2-2b-it/hotpot_qa # 90k
- self_gen/google/gemma-2-2b-it/squad # 90k
- self_gen/google/gemma-2-2b-it/drop # 77k
- self_gen/google/gemma-2-2b-it/narrativeqa # 40k
- self_gen/google/gemma-2-2b-it/quoref # 11k
- self_gen/google/gemma-2-2b-it/ropes # 11k
- self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

49
configs/fw_qa_pretrain_small_and_pwc.yaml
View file

@ -1,49 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small_pretrain
- self_gen/google/gemma-2-2b-it/pwc
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/pwc
- pwc
load_best_model_at_end: true
metric_for_best_model: eval_fw_qa_3_pretrain_loss

18
configs/fw_qa_pretrain_mini.yaml → configs/fw_qa_v2_level_0_tiny.yaml
View file

@ -1,6 +1,6 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
model_name_or_path: google/gemma-3-1b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
@ -23,24 +23,24 @@ max_val_samples_per_ds: 1000
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
neftune_noise_alpha: 1
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_prefetch_factor: 16
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_mini_pretrain
- fw_qa_v2_2k_len_level_0_tiny
val_ds_names:
- fw_qa_3_pretrain
load_best_model_at_end: true
metric_for_best_model: eval_fw_qa_3_pretrain_loss
- fw_qa_v2_2k_len_level_0
- squad

18
configs/fw_qa_pretrain_small.yaml → configs/fw_qa_v2_level_3.yaml
View file

@ -1,6 +1,6 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
model_name_or_path: google/gemma-3-1b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
@ -23,24 +23,24 @@ max_val_samples_per_ds: 1000
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
neftune_noise_alpha: 1
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_prefetch_factor: 16
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small_pretrain
- fw_qa_v2_2k_len_level_3
val_ds_names:
- fw_qa_3_pretrain
load_best_model_at_end: true
metric_for_best_model: eval_fw_qa_3_pretrain_loss
- fw_qa_v2_2k_len_level_0
- squad

13
configs/self_gen_pwc_hotpot_qa.yaml → configs/fw_qa_v2_level_3_tiny.yaml
View file

@ -1,6 +1,6 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
model_name_or_path: google/gemma-3-1b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
@ -28,20 +28,19 @@ weight_decay: 0.01
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_prefetch_factor: 16
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.02
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- fw_qa_v2_2k_len_level_3_tiny
val_ds_names:
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- fw_qa_v2_2k_len_level_0
- squad

65
configs/qa.yaml
View file

@ -1,65 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small # ~ 20M
- ctx_qa # 300k
- pwc # 240k
- hotpot_qa # 90k
- squad # 90k
- drop # 77k
- narrativeqa # 40k
- quoref # 11k
- ropes # 11k
- synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

59
configs/qa_no_fw.yaml
View file

@ -1,59 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- ctx_qa # 300k
- pwc # 240k
- hotpot_qa # 90k
- squad # 90k
- drop # 77k
- narrativeqa # 40k
- quoref # 11k
- ropes # 11k
- synthetic_convqa # 40k
val_ds_names:
- fw_qa_3
- fw_qa_xl
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: true
metric_for_best_model: eval_pwc_loss

66
configs/self_gen_3_and_pretrain_small.yaml
View file

@ -1,66 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_small_pretrain # ~20M
- self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M
- self_gen/google/gemma-2-2b-it/ctx_qa # 300k
- self_gen/google/gemma-2-2b-it/pwc # 240k
- self_gen/google/gemma-2-2b-it/hotpot_qa # 90k
- self_gen/google/gemma-2-2b-it/squad # 90k
- self_gen/google/gemma-2-2b-it/drop # 77k
- self_gen/google/gemma-2-2b-it/narrativeqa # 40k
- self_gen/google/gemma-2-2b-it/quoref # 11k
- self_gen/google/gemma-2-2b-it/ropes # 11k
- self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

67
configs/self_gen_3_mini.yaml
View file

@ -1,67 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it/fw_qa_3_mini # 100k
- self_gen/google/gemma-2-2b-it/ctx_qa # 300k
- self_gen/google/gemma-2-2b-it/pwc # 240k
- self_gen/google/gemma-2-2b-it/hotpot_qa # 90k
- self_gen/google/gemma-2-2b-it/squad # 90k
- self_gen/google/gemma-2-2b-it/drop # 77k
- self_gen/google/gemma-2-2b-it/narrativeqa # 40k
- self_gen/google/gemma-2-2b-it/quoref # 11k
- self_gen/google/gemma-2-2b-it/ropes # 11k
- self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k
val_ds_names:
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/fw_qa_xl
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- fw_qa_xl
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: true
metric_for_best_model: eval_pwc_loss
# metric_for_best_model: eval_fw_qa_3_loss

65
configs/self_gen_3_small.yaml
View file

@ -1,65 +0,0 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
# eval_steps: 500
# save_strategy: "no"
# # save_steps: 500
# logging_strategy: "steps"
# logging_steps: 100
# use_liger_kernel: true
# remove_unused_columns: false
# needed to avoid OOM by compute the metrics batch by batch
# w/o this the trainer stores logits of all sample in memory...
# batch_eval_metrics: true
per_device_train_batch_size: 8
per_device_eval_batch_size: 8
max_val_samples_per_ds: 1000
# optim: schedule_free_adamw
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M
- self_gen/google/gemma-2-2b-it/ctx_qa # 300k
- self_gen/google/gemma-2-2b-it/pwc # 240k
- self_gen/google/gemma-2-2b-it/hotpot_qa # 90k
- self_gen/google/gemma-2-2b-it/squad # 90k
- self_gen/google/gemma-2-2b-it/drop # 77k
- self_gen/google/gemma-2-2b-it/narrativeqa # 40k
- self_gen/google/gemma-2-2b-it/quoref # 11k
- self_gen/google/gemma-2-2b-it/ropes # 11k
- self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
- self_gen/google/gemma-2-2b-it/ctx_qa
- self_gen/google/gemma-2-2b-it/pwc
- self_gen/google/gemma-2-2b-it/hotpot_qa
- self_gen/google/gemma-2-2b-it/squad
- fw_qa_3
- ctx_qa
- pwc
- hotpot_qa
- squad
load_best_model_at_end: false
metric_for_best_model: eval_fw_qa_3_pretrain_loss

22
configs/fw_qa_pretrain_mini_and_pwc.yaml → configs/squad_fw_qa_v2_level_3.yaml
View file

@ -1,6 +1,6 @@
output_dir: "" # just a placeholder
bf16: true
model_name_or_path: google/gemma-2-2b-it
model_name_or_path: google/gemma-3-1b-it
label_names: ["labels"]
# eval_on_start: True
# eval_strategy: "steps"
@ -23,27 +23,25 @@ max_val_samples_per_ds: 1000
learning_rate: 0.00004
# lr_scheduler_type: "constant_with_warmup"
neftune_noise_alpha: 5
neftune_noise_alpha: 1
weight_decay: 0.01
#
warmup_steps: 100
dataloader_prefetch_factor: 8
dataloader_prefetch_factor: 16
dataloader_num_workers: 8
# LoRA
lora_r: 8
lora_dropout: 0.0
target_modules:
- down_proj
# data
train_ds_names:
- fw_qa_3_mini_pretrain
- self_gen/google/gemma-2-2b-it/pwc
- squad
- fw_qa_v2_2k_len_level_3
val_ds_names:
- fw_qa_3_pretrain
- self_gen/google/gemma-2-2b-it/pwc
- pwc
load_best_model_at_end: true
metric_for_best_model: eval_fw_qa_3_pretrain_loss
- fw_qa_v2_2k_len_level_0
- squad

5
install.sh
View file

@ -1,4 +1,4 @@
# uv pip install -e .
# uv pip install -e .
# uv pip install flash-attn==2.6.3 --no-build-isolation
# # huggingface-cli login
@ -26,5 +26,8 @@ uv pip install flashinfer-python==0.2.2 -i https://flashinfer.ai/whl/cu124/torch
# needed for gated models
uv run huggingface-cli login
# needed for logging with wandb
wandb login
# dev
uv run pre-commit install

35
scripts/fw_qa_pretrain/gemma_pretrain.sh
View file

@ -1,35 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
# module load
# module load cuda/12.1
# module load cudnn/8.9.7
# module load nccl/cuda-12.1/2.18.3
# module load hpcx/2.20
# export OMP_NUM_THREADS=24
# export TRITON_CACHE_DIR=/tmp/.triton/
# . ~/miniconda3/etc/profile.d/conda.sh
# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
# eval "$@"
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29560 intx_sft.py configs/fw_qa_pretrain_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=4 \
--gradient_accumulation_steps=32 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

35
scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh
View file

@ -1,35 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
# module load
# module load cuda/12.1
# module load cudnn/8.9.7
# module load nccl/cuda-12.1/2.18.3
# module load hpcx/2.20
# export OMP_NUM_THREADS=24
# export TRITON_CACHE_DIR=/tmp/.triton/
# . ~/miniconda3/etc/profile.d/conda.sh
# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
# eval "$@"
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29560 intx_sft.py configs/fw_qa_pretrain_small_and_pwc.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=4 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

25
scripts/gemma_data_exp/gemma_qa.sh
View file

@ -1,25 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29561 intx_sft.py configs/qa.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

25
scripts/gemma_data_exp/gemma_qa_8_gpu.sh
View file

@ -1,25 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=8
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29561 intx_sft.py configs/qa.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=8 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=16 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

24
scripts/gemma_data_exp/gemma_self_gen_qa.sh
View file

@ -1,24 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

24
scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh
View file

@ -1,24 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_mini.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=3 --per_device_train_batch_size=8 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

26
scripts/gemma_data_exp/gemma_self_gen_qa_repeat.sh
View file

@ -1,26 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
# NO ! #SBATCH --begin=now+6hour # delay scheduling
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29563 intx_sft.py configs/self_gen_3_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=True --repeat_prob=0.1 \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

26
scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain.sh
View file

@ -1,26 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
# NO ! #SBATCH --begin=now+6hour # delay scheduling
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29564 intx_sft.py configs/data_exp/self_gen_3_and_pretrain_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1.5 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=True --repeat_prob=0.1 \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

26
scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain_augment.sh
View file

@ -1,26 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
# NO ! #SBATCH --begin=now+6hour # delay scheduling
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29565 intx_sft.py configs/data_exp/self_gen_3_and_pretrain_small_aug.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \
--lora_r=8 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=True --repeat_prob=0.1 \
--use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

24
scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16.sh
View file

@ -1,24 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=8
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29561 intx_sft.py configs/self_gen_3_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=1 \
--lora_r=16 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

26
scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16_lightweight.sh
View file

@ -1,26 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29563 intx_sft.py configs/self_gen_3_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=32 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=1 \
--lora_r=16 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--num_pre_head_layers=1 \
--latent_size=256 \
--neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

24
scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16.sh
View file

@ -1,24 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=8
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=1 \
--lora_r=16 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--add_negative_prompt=False \
--add_repeat_prompt=True --repeat_prob=0.1 \
--use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

26
scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16_lightweight.sh
View file

@ -1,26 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29564 intx_sft.py configs/self_gen_3_small.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=-1 \
--gradient_accumulation_steps=32 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=1 \
--lora_r=16 \
--eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \
--save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--num_pre_head_layers=1 \
--latent_size=256 \
--neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \
--add_negative_prompt=False \
--add_repeat_prompt=True --repeat_prob=0.1 \
--use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

35
scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh
View file

@ -1,35 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
# module load
# module load cuda/12.1
# module load cudnn/8.9.7
# module load nccl/cuda-12.1/2.18.3
# module load hpcx/2.20
# export OMP_NUM_THREADS=24
# export TRITON_CACHE_DIR=/tmp/.triton/
# . ~/miniconda3/etc/profile.d/conda.sh
# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
# eval "$@"
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29564 intx_sft.py configs/self_gen_3_mini.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10 --per_device_train_batch_size=24 \
--gradient_accumulation_steps=8 --per_device_eval_batch_size=24 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=4 --num_latent_factor=1 \
--lora_r=8 \
--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True --per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

35
scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh
View file

@ -1,35 +0,0 @@
#!/bin/bash
#SBATCH --job-name=ctxlora_medium
#SBATCH --partition=a3
#SBATCH --nodes=1
#SBATCH --gpus=4
#SBATCH --output=outputs/%x-%j.out
#SBATCH --error=outputs/%x-%j.out
# module load
# module load cuda/12.1
# module load cudnn/8.9.7
# module load nccl/cuda-12.1/2.18.3
# module load hpcx/2.20
# export OMP_NUM_THREADS=24
# export TRITON_CACHE_DIR=/tmp/.triton/
# . ~/miniconda3/etc/profile.d/conda.sh
# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
# eval "$@"
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29560 intx_sft.py configs/self_gen_3_mini.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10 --per_device_train_batch_size=24 \
--gradient_accumulation_steps=8 --per_device_eval_batch_size=24 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=4 --num_latent_factor=1 \
--lora_r=8 \
--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
--neftune_noise_alpha=5 --use_light_weight_lora=False \
--load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \
--add_repeat_prompt=True --repeat_prob=0.1 \
--use_sequence_packing=True --per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \

34
scripts/short_ctx/gemma_fw_qa_level_0_tiny.sh Normal file
View file

@ -0,0 +1,34 @@
#!/bin/bash
#SBATCH --job-name=ctxlora
#SBATCH --nodes=1
#SBATCH --partition=sakura-gpu
#SBATCH --gpus=4
#SBATCH --output=slurm_logs/%x-%j.out
#SBATCH --error=slurm_logs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29562 intx_sft.py configs/fw_qa_v2_level_0_tiny.yaml \
--model_name_or_path=google/gemma-3-1b-it \
--num_train_epochs=5 \
--per_device_train_batch_size=-1 \
--gradient_accumulation_steps=2 \
--per_device_eval_batch_size=64 \
--target_modules=down_proj \
--num_self_attends_per_block=8 \
--num_latent_factor=1 \
--num_pre_head_layers=1 \
--lora_r=8 \
--eval_steps=1000 \
--save_steps=1000 \
--learning_rate=4e-5 \
--lora_dropout=0.0 \
--neftune_noise_alpha=5 \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True \
--max_packed_inp_len=16000 \
--max_packed_ctx_len=32000 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \
--logging_steps=50

34
scripts/short_ctx/gemma_fw_qa_level_3.sh Normal file
View file

@ -0,0 +1,34 @@
#!/bin/bash
#SBATCH --job-name=ctxlora
#SBATCH --nodes=1
#SBATCH --partition=sakura-gpu
#SBATCH --gpus=4
#SBATCH --output=slurm_logs/%x-%j.out
#SBATCH --error=slurm_logs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29564 intx_sft.py configs/fw_qa_v2_level_3.yaml \
--model_name_or_path=google/gemma-3-1b-it \
--num_train_epochs=5 \
--per_device_train_batch_size=-1 \
--gradient_accumulation_steps=2 \
--per_device_eval_batch_size=64 \
--target_modules=down_proj \
--num_self_attends_per_block=8 \
--num_latent_factor=1 \
--num_pre_head_layers=1 \
--lora_r=8 \
--eval_steps=1000 \
--save_steps=1000 \
--learning_rate=4e-5 \
--lora_dropout=0.0 \
--neftune_noise_alpha=5 \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True \
--max_packed_inp_len=16000 \
--max_packed_ctx_len=32000 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \
--logging_steps=50

34
scripts/short_ctx/gemma_fw_qa_level_3_tiny.sh Normal file
View file

@ -0,0 +1,34 @@
#!/bin/bash
#SBATCH --job-name=ctxlora
#SBATCH --nodes=1
#SBATCH --partition=sakura-gpu
#SBATCH --gpus=4
#SBATCH --output=slurm_logs/%x-%j.out
#SBATCH --error=slurm_logs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29563 intx_sft.py configs/fw_qa_v2_level_3_tiny.yaml \
--model_name_or_path=google/gemma-3-1b-it \
--num_train_epochs=5 \
--per_device_train_batch_size=-1 \
--gradient_accumulation_steps=2 \
--per_device_eval_batch_size=64 \
--target_modules=down_proj \
--num_self_attends_per_block=8 \
--num_latent_factor=1 \
--num_pre_head_layers=1 \
--lora_r=8 \
--eval_steps=1000 \
--save_steps=1000 \
--learning_rate=4e-5 \
--lora_dropout=0.0 \
--neftune_noise_alpha=5 \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True \
--max_packed_inp_len=16000 \
--max_packed_ctx_len=32000 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \
--logging_steps=50

34
scripts/short_ctx/gemma_squad.sh Normal file
View file

@ -0,0 +1,34 @@
#!/bin/bash
#SBATCH --job-name=ctxlora
#SBATCH --nodes=1
#SBATCH --partition=sakura-gpu
#SBATCH --gpus=4
#SBATCH --output=slurm_logs/%x-%j.out
#SBATCH --error=slurm_logs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29561 intx_sft.py configs/squad.yaml \
--model_name_or_path=google/gemma-3-1b-it \
--num_train_epochs=5 \
--per_device_train_batch_size=-1 \
--gradient_accumulation_steps=2 \
--per_device_eval_batch_size=64 \
--target_modules=down_proj \
--num_self_attends_per_block=8 \
--num_latent_factor=1 \
--num_pre_head_layers=1 \
--lora_r=8 \
--eval_steps=1000 \
--save_steps=1000 \
--learning_rate=4e-5 \
--lora_dropout=0.0 \
--neftune_noise_alpha=5 \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True \
--max_packed_inp_len=16000 \
--max_packed_ctx_len=32000 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \
--logging_steps=50

34
scripts/short_ctx/gemma_squad_and_fw_qa_level_3.sh Normal file
View file

@ -0,0 +1,34 @@
#!/bin/bash
#SBATCH --job-name=ctxlora
#SBATCH --nodes=1
#SBATCH --partition=sakura-gpu
#SBATCH --gpus=4
#SBATCH --output=slurm_logs/%x-%j.out
#SBATCH --error=slurm_logs/%x-%j.out
uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29565 intx_sft.py configs/squad.yaml \
--model_name_or_path=google/gemma-3-1b-it \
--num_train_epochs=5 \
--per_device_train_batch_size=-1 \
--gradient_accumulation_steps=2 \
--per_device_eval_batch_size=64 \
--target_modules=down_proj \
--num_self_attends_per_block=8 \
--num_latent_factor=1 \
--num_pre_head_layers=1 \
--lora_r=8 \
--eval_steps=1000 \
--save_steps=1000 \
--learning_rate=4e-5 \
--lora_dropout=0.0 \
--neftune_noise_alpha=5 \
--add_negative_prompt=False \
--add_repeat_prompt=False \
--use_sequence_packing=True \
--max_packed_inp_len=16000 \
--max_packed_ctx_len=32000 \
--per_rank_gen=True \
--per_layer_processing=True \
--gen_lora_l1_reg_coef=0.1 \
--logging_steps=50

2
src/ctx_to_lora/configs.py
View file

@ -448,7 +448,7 @@ class CtxEncoderArguments:
@dataclass
class AggregatorArguments:
aggregator_type: Literal["pooler", "perceiver"] = field(
default="pooler",
default="perceiver",
metadata={"help": "Aggregator type for HyperLoRA."},
)