diff --git a/README.md b/README.md index 2897d27..8470f20 100644 --- a/README.md +++ b/README.md @@ -33,9 +33,14 @@ print(model.decode(outputs)) ```bash DEBUG=1 WANDB_MODE=disabled uv run intx_sft.py configs/context_numbers_10.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=1 --per_device_train_batch_size=64 --gradient_accumulation_steps=1 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=2 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=False --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --use_sequence_packing=True --max_packed_inp_len=8000 --max_packed_ctx_len=16000 --dataloader_num_workers=0 --dataloader_prefetch_factor=None --eval_on_start=True ``` -### Squad + Hotpot training (for testing/debugging) +### FWQA-v2 Level-0 Tiny ```bash -WANDB_MODE=disabled uv run intx_sft.py configs/squad.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --metric_for_best_model=eval_pwc_loss --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=16000 --max_packed_ctx_len=32000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=10 +WANDB_MODE=disabled run uv run intx_sft.py configs/fw_qa_v2_level_0_tiny.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=24000 --max_packed_ctx_len=48000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=50 +``` + +### Squad only +```bash +WANDB_MODE=disabled uv run intx_sft.py configs/squad.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=16000 --max_packed_ctx_len=32000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=10 ``` ### HyperLoRA w/ self-gen 3 mini diff --git a/configs/data_exp/qa.yaml b/configs/data_exp/qa.yaml deleted file mode 100644 index c699d9b..0000000 --- a/configs/data_exp/qa.yaml +++ /dev/null @@ -1,65 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_small # ~ 20M - - ctx_qa # 300k - - pwc # 240k - - hotpot_qa # 90k - - squad # 90k - - drop # 77k - - narrativeqa # 40k - - quoref # 11k - - ropes # 11k - - synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/data_exp/self_gen_3_and_pretrain_small.yaml b/configs/data_exp/self_gen_3_and_pretrain_small.yaml deleted file mode 100644 index 5d128e3..0000000 --- a/configs/data_exp/self_gen_3_and_pretrain_small.yaml +++ /dev/null @@ -1,66 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_small_pretrain # ~6M - - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M - - self_gen/google/gemma-2-2b-it/ctx_qa # 300k - - self_gen/google/gemma-2-2b-it/pwc # 240k - - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k - - self_gen/google/gemma-2-2b-it/squad # 90k - - self_gen/google/gemma-2-2b-it/drop # 77k - - self_gen/google/gemma-2-2b-it/narrativeqa # 40k - - self_gen/google/gemma-2-2b-it/quoref # 11k - - self_gen/google/gemma-2-2b-it/ropes # 11k - - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/data_exp/self_gen_3_and_pretrain_small_aug.yaml b/configs/data_exp/self_gen_3_and_pretrain_small_aug.yaml deleted file mode 100644 index 9cb3931..0000000 --- a/configs/data_exp/self_gen_3_and_pretrain_small_aug.yaml +++ /dev/null @@ -1,67 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_small_pretrain # ~6M - - fw_qa_3_small_aug_pretrain # ~12M - - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M - - self_gen/google/gemma-2-2b-it/ctx_qa # 300k - - self_gen/google/gemma-2-2b-it/pwc # 240k - - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k - - self_gen/google/gemma-2-2b-it/squad # 90k - - self_gen/google/gemma-2-2b-it/drop # 77k - - self_gen/google/gemma-2-2b-it/narrativeqa # 40k - - self_gen/google/gemma-2-2b-it/quoref # 11k - - self_gen/google/gemma-2-2b-it/ropes # 11k - - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/data_exp/self_gen_3_small.yaml b/configs/data_exp/self_gen_3_small.yaml deleted file mode 100644 index a8580a3..0000000 --- a/configs/data_exp/self_gen_3_small.yaml +++ /dev/null @@ -1,65 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M - - self_gen/google/gemma-2-2b-it/ctx_qa # 300k - - self_gen/google/gemma-2-2b-it/pwc # 240k - - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k - - self_gen/google/gemma-2-2b-it/squad # 90k - - self_gen/google/gemma-2-2b-it/drop # 77k - - self_gen/google/gemma-2-2b-it/narrativeqa # 40k - - self_gen/google/gemma-2-2b-it/quoref # 11k - - self_gen/google/gemma-2-2b-it/ropes # 11k - - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/fw_qa_pretrain_small_and_pwc.yaml b/configs/fw_qa_pretrain_small_and_pwc.yaml deleted file mode 100644 index bc3ea17..0000000 --- a/configs/fw_qa_pretrain_small_and_pwc.yaml +++ /dev/null @@ -1,49 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_small_pretrain - - self_gen/google/gemma-2-2b-it/pwc - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/pwc - - pwc - -load_best_model_at_end: true -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/fw_qa_pretrain_mini.yaml b/configs/fw_qa_v2_level_0_tiny.yaml similarity index 78% rename from configs/fw_qa_pretrain_mini.yaml rename to configs/fw_qa_v2_level_0_tiny.yaml index 0c7a92b..051f7f3 100644 --- a/configs/fw_qa_pretrain_mini.yaml +++ b/configs/fw_qa_v2_level_0_tiny.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: google/gemma-2-2b-it +model_name_or_path: google/gemma-3-1b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -23,24 +23,24 @@ max_val_samples_per_ds: 1000 learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 +neftune_noise_alpha: 1 weight_decay: 0.01 -# + warmup_steps: 100 -dataloader_prefetch_factor: 8 +dataloader_prefetch_factor: 16 dataloader_num_workers: 8 + # LoRA lora_r: 8 lora_dropout: 0.0 target_modules: - down_proj + # data train_ds_names: - - fw_qa_3_mini_pretrain + - fw_qa_v2_2k_len_level_0_tiny val_ds_names: - - fw_qa_3_pretrain - -load_best_model_at_end: true -metric_for_best_model: eval_fw_qa_3_pretrain_loss + - fw_qa_v2_2k_len_level_0 + - squad diff --git a/configs/fw_qa_pretrain_small.yaml b/configs/fw_qa_v2_level_3.yaml similarity index 78% rename from configs/fw_qa_pretrain_small.yaml rename to configs/fw_qa_v2_level_3.yaml index 4224c8a..34f9149 100644 --- a/configs/fw_qa_pretrain_small.yaml +++ b/configs/fw_qa_v2_level_3.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: google/gemma-2-2b-it +model_name_or_path: google/gemma-3-1b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -23,24 +23,24 @@ max_val_samples_per_ds: 1000 learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 +neftune_noise_alpha: 1 weight_decay: 0.01 -# + warmup_steps: 100 -dataloader_prefetch_factor: 8 +dataloader_prefetch_factor: 16 dataloader_num_workers: 8 + # LoRA lora_r: 8 lora_dropout: 0.0 target_modules: - down_proj + # data train_ds_names: - - fw_qa_3_small_pretrain + - fw_qa_v2_2k_len_level_3 val_ds_names: - - fw_qa_3_pretrain - -load_best_model_at_end: true -metric_for_best_model: eval_fw_qa_3_pretrain_loss + - fw_qa_v2_2k_len_level_0 + - squad diff --git a/configs/self_gen_pwc_hotpot_qa.yaml b/configs/fw_qa_v2_level_3_tiny.yaml similarity index 75% rename from configs/self_gen_pwc_hotpot_qa.yaml rename to configs/fw_qa_v2_level_3_tiny.yaml index 6da0f94..9f25503 100644 --- a/configs/self_gen_pwc_hotpot_qa.yaml +++ b/configs/fw_qa_v2_level_3_tiny.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: google/gemma-2-2b-it +model_name_or_path: google/gemma-3-1b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -28,20 +28,19 @@ weight_decay: 0.01 warmup_steps: 100 -dataloader_prefetch_factor: 8 +dataloader_prefetch_factor: 16 dataloader_num_workers: 8 # LoRA lora_r: 8 -lora_dropout: 0.02 +lora_dropout: 0.0 target_modules: - down_proj # data train_ds_names: - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa + - fw_qa_v2_2k_len_level_3_tiny val_ds_names: - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa + - fw_qa_v2_2k_len_level_0 + - squad diff --git a/configs/qa.yaml b/configs/qa.yaml deleted file mode 100644 index c699d9b..0000000 --- a/configs/qa.yaml +++ /dev/null @@ -1,65 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_small # ~ 20M - - ctx_qa # 300k - - pwc # 240k - - hotpot_qa # 90k - - squad # 90k - - drop # 77k - - narrativeqa # 40k - - quoref # 11k - - ropes # 11k - - synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/qa_no_fw.yaml b/configs/qa_no_fw.yaml deleted file mode 100644 index e59f536..0000000 --- a/configs/qa_no_fw.yaml +++ /dev/null @@ -1,59 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - ctx_qa # 300k - - pwc # 240k - - hotpot_qa # 90k - - squad # 90k - - drop # 77k - - narrativeqa # 40k - - quoref # 11k - - ropes # 11k - - synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3 - - fw_qa_xl - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/self_gen_3_and_pretrain_small.yaml b/configs/self_gen_3_and_pretrain_small.yaml deleted file mode 100644 index 7184ea1..0000000 --- a/configs/self_gen_3_and_pretrain_small.yaml +++ /dev/null @@ -1,66 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_small_pretrain # ~20M - - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M - - self_gen/google/gemma-2-2b-it/ctx_qa # 300k - - self_gen/google/gemma-2-2b-it/pwc # 240k - - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k - - self_gen/google/gemma-2-2b-it/squad # 90k - - self_gen/google/gemma-2-2b-it/drop # 77k - - self_gen/google/gemma-2-2b-it/narrativeqa # 40k - - self_gen/google/gemma-2-2b-it/quoref # 11k - - self_gen/google/gemma-2-2b-it/ropes # 11k - - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/self_gen_3_mini.yaml b/configs/self_gen_3_mini.yaml deleted file mode 100644 index 678c1e8..0000000 --- a/configs/self_gen_3_mini.yaml +++ /dev/null @@ -1,67 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - self_gen/google/gemma-2-2b-it/fw_qa_3_mini # 100k - - self_gen/google/gemma-2-2b-it/ctx_qa # 300k - - self_gen/google/gemma-2-2b-it/pwc # 240k - - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k - - self_gen/google/gemma-2-2b-it/squad # 90k - - self_gen/google/gemma-2-2b-it/drop # 77k - - self_gen/google/gemma-2-2b-it/narrativeqa # 40k - - self_gen/google/gemma-2-2b-it/quoref # 11k - - self_gen/google/gemma-2-2b-it/ropes # 11k - - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k - -val_ds_names: - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/fw_qa_xl - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - fw_qa_xl - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss -# metric_for_best_model: eval_fw_qa_3_loss diff --git a/configs/self_gen_3_small.yaml b/configs/self_gen_3_small.yaml deleted file mode 100644 index a8580a3..0000000 --- a/configs/self_gen_3_small.yaml +++ /dev/null @@ -1,65 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-2-2b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M - - self_gen/google/gemma-2-2b-it/ctx_qa # 300k - - self_gen/google/gemma-2-2b-it/pwc # 240k - - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k - - self_gen/google/gemma-2-2b-it/squad # 90k - - self_gen/google/gemma-2-2b-it/drop # 77k - - self_gen/google/gemma-2-2b-it/narrativeqa # 40k - - self_gen/google/gemma-2-2b-it/quoref # 11k - - self_gen/google/gemma-2-2b-it/ropes # 11k - - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/fw_qa_3_small - - self_gen/google/gemma-2-2b-it/ctx_qa - - self_gen/google/gemma-2-2b-it/pwc - - self_gen/google/gemma-2-2b-it/hotpot_qa - - self_gen/google/gemma-2-2b-it/squad - - fw_qa_3 - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: false -metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/fw_qa_pretrain_mini_and_pwc.yaml b/configs/squad_fw_qa_v2_level_3.yaml similarity index 72% rename from configs/fw_qa_pretrain_mini_and_pwc.yaml rename to configs/squad_fw_qa_v2_level_3.yaml index c392f03..04b7dd5 100644 --- a/configs/fw_qa_pretrain_mini_and_pwc.yaml +++ b/configs/squad_fw_qa_v2_level_3.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: google/gemma-2-2b-it +model_name_or_path: google/gemma-3-1b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -23,27 +23,25 @@ max_val_samples_per_ds: 1000 learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 +neftune_noise_alpha: 1 weight_decay: 0.01 -# + warmup_steps: 100 -dataloader_prefetch_factor: 8 +dataloader_prefetch_factor: 16 dataloader_num_workers: 8 + # LoRA lora_r: 8 lora_dropout: 0.0 target_modules: - down_proj + # data train_ds_names: - - fw_qa_3_mini_pretrain - - self_gen/google/gemma-2-2b-it/pwc + - squad + - fw_qa_v2_2k_len_level_3 val_ds_names: - - fw_qa_3_pretrain - - self_gen/google/gemma-2-2b-it/pwc - - pwc - -load_best_model_at_end: true -metric_for_best_model: eval_fw_qa_3_pretrain_loss + - fw_qa_v2_2k_len_level_0 + - squad diff --git a/install.sh b/install.sh index cac146b..167f521 100755 --- a/install.sh +++ b/install.sh @@ -1,4 +1,4 @@ -# uv pip install -e . +# uv pip install -e . # uv pip install flash-attn==2.6.3 --no-build-isolation # # huggingface-cli login @@ -26,5 +26,8 @@ uv pip install flashinfer-python==0.2.2 -i https://flashinfer.ai/whl/cu124/torch # needed for gated models uv run huggingface-cli login +# needed for logging with wandb +wandb login + # dev uv run pre-commit install diff --git a/scripts/fw_qa_pretrain/gemma_pretrain.sh b/scripts/fw_qa_pretrain/gemma_pretrain.sh deleted file mode 100644 index 1ce7fb6..0000000 --- a/scripts/fw_qa_pretrain/gemma_pretrain.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -# . ~/miniconda3/etc/profile.d/conda.sh -# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29560 intx_sft.py configs/fw_qa_pretrain_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=4 \ ---gradient_accumulation_steps=32 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh b/scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh deleted file mode 100644 index 067937a..0000000 --- a/scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -# . ~/miniconda3/etc/profile.d/conda.sh -# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29560 intx_sft.py configs/fw_qa_pretrain_small_and_pwc.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=4 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_data_exp/gemma_qa.sh b/scripts/gemma_data_exp/gemma_qa.sh deleted file mode 100644 index 6521635..0000000 --- a/scripts/gemma_data_exp/gemma_qa.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29561 intx_sft.py configs/qa.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_data_exp/gemma_qa_8_gpu.sh b/scripts/gemma_data_exp/gemma_qa_8_gpu.sh deleted file mode 100644 index e4c7636..0000000 --- a/scripts/gemma_data_exp/gemma_qa_8_gpu.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=8 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29561 intx_sft.py configs/qa.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=16 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa.sh b/scripts/gemma_data_exp/gemma_self_gen_qa.sh deleted file mode 100644 index 1562989..0000000 --- a/scripts/gemma_data_exp/gemma_self_gen_qa.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh deleted file mode 100644 index d058bda..0000000 --- a/scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_mini.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=3 --per_device_train_batch_size=8 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_repeat.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_repeat.sh deleted file mode 100644 index 639d6f2..0000000 --- a/scripts/gemma_data_exp/gemma_self_gen_qa_repeat.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -# NO ! #SBATCH --begin=now+6hour # delay scheduling -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=True --repeat_prob=0.1 \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain.sh deleted file mode 100644 index 9b646fe..0000000 --- a/scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -# NO ! #SBATCH --begin=now+6hour # delay scheduling -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/data_exp/self_gen_3_and_pretrain_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1.5 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=True --repeat_prob=0.1 \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain_augment.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain_augment.sh deleted file mode 100644 index 104a178..0000000 --- a/scripts/gemma_data_exp/gemma_self_gen_qa_repeat_and_pretrain_augment.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -# NO ! #SBATCH --begin=now+6hour # delay scheduling -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29565 intx_sft.py configs/data_exp/self_gen_3_and_pretrain_small_aug.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=True --repeat_prob=0.1 \ ---use_sequence_packing=True --max_packed_inp_len=8192 --max_packed_ctx_len=16384 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16.sh b/scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16.sh deleted file mode 100644 index 3871e79..0000000 --- a/scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=8 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29561 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=1 \ ---lora_r=16 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ diff --git a/scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16_lightweight.sh b/scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16_lightweight.sh deleted file mode 100644 index 6df3fb5..0000000 --- a/scripts/gemma_rank_exp/gemma_self_gen_qa_rank_16_lightweight.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=32 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=1 \ ---lora_r=16 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---num_pre_head_layers=1 \ ---latent_size=256 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ diff --git a/scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16.sh b/scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16.sh deleted file mode 100644 index 7515897..0000000 --- a/scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=8 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2.2 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=1 \ ---lora_r=16 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---add_negative_prompt=False \ ---add_repeat_prompt=True --repeat_prob=0.1 \ ---use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ diff --git a/scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16_lightweight.sh b/scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16_lightweight.sh deleted file mode 100644 index ad39662..0000000 --- a/scripts/gemma_rank_exp/gemma_self_gen_qa_repeat_rank_16_lightweight.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=32 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=1 \ ---lora_r=16 \ ---eval_steps=1000000000000000 --do_eval=no --do_predict=no --eval_on_start=False \ ---save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---num_pre_head_layers=1 \ ---latent_size=256 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---add_negative_prompt=False \ ---add_repeat_prompt=True --repeat_prob=0.1 \ ---use_sequence_packing=True --max_packed_inp_len=4096 --max_packed_ctx_len=8192 \ ---per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ diff --git a/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh b/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh deleted file mode 100644 index 8d2c34c..0000000 --- a/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -# . ~/miniconda3/etc/profile.d/conda.sh -# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/self_gen_3_mini.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10 --per_device_train_batch_size=24 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=24 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=4 --num_latent_factor=1 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh b/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh deleted file mode 100644 index ba0fc8d..0000000 --- a/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -# . ~/miniconda3/etc/profile.d/conda.sh -# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29560 intx_sft.py configs/self_gen_3_mini.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10 --per_device_train_batch_size=24 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=24 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=4 --num_latent_factor=1 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=True --repeat_prob=0.1 \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ - diff --git a/scripts/short_ctx/gemma_fw_qa_level_0_tiny.sh b/scripts/short_ctx/gemma_fw_qa_level_0_tiny.sh new file mode 100644 index 0000000..d5ada17 --- /dev/null +++ b/scripts/short_ctx/gemma_fw_qa_level_0_tiny.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29562 intx_sft.py configs/fw_qa_v2_level_0_tiny.yaml \ +--model_name_or_path=google/gemma-3-1b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file diff --git a/scripts/short_ctx/gemma_fw_qa_level_3.sh b/scripts/short_ctx/gemma_fw_qa_level_3.sh new file mode 100644 index 0000000..99ccaf4 --- /dev/null +++ b/scripts/short_ctx/gemma_fw_qa_level_3.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29564 intx_sft.py configs/fw_qa_v2_level_3.yaml \ +--model_name_or_path=google/gemma-3-1b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file diff --git a/scripts/short_ctx/gemma_fw_qa_level_3_tiny.sh b/scripts/short_ctx/gemma_fw_qa_level_3_tiny.sh new file mode 100644 index 0000000..640286f --- /dev/null +++ b/scripts/short_ctx/gemma_fw_qa_level_3_tiny.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29563 intx_sft.py configs/fw_qa_v2_level_3_tiny.yaml \ +--model_name_or_path=google/gemma-3-1b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file diff --git a/scripts/short_ctx/gemma_squad.sh b/scripts/short_ctx/gemma_squad.sh new file mode 100644 index 0000000..647e8e7 --- /dev/null +++ b/scripts/short_ctx/gemma_squad.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29561 intx_sft.py configs/squad.yaml \ +--model_name_or_path=google/gemma-3-1b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file diff --git a/scripts/short_ctx/gemma_squad_and_fw_qa_level_3.sh b/scripts/short_ctx/gemma_squad_and_fw_qa_level_3.sh new file mode 100644 index 0000000..848d5fd --- /dev/null +++ b/scripts/short_ctx/gemma_squad_and_fw_qa_level_3.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29565 intx_sft.py configs/squad.yaml \ +--model_name_or_path=google/gemma-3-1b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file diff --git a/src/ctx_to_lora/configs.py b/src/ctx_to_lora/configs.py index d2e9f14..afe8a06 100644 --- a/src/ctx_to_lora/configs.py +++ b/src/ctx_to_lora/configs.py @@ -448,7 +448,7 @@ class CtxEncoderArguments: @dataclass class AggregatorArguments: aggregator_type: Literal["pooler", "perceiver"] = field( - default="pooler", + default="perceiver", metadata={"help": "Aggregator type for HyperLoRA."}, )