diff --git a/configs/pwc_hotpot_qa.yaml b/configs/pwc_hotpot_qa.yaml new file mode 100644 index 0000000..26da9d3 --- /dev/null +++ b/configs/pwc_hotpot_qa.yaml @@ -0,0 +1,51 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-2-2b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.01 + +warmup_steps: 100 + +dataloader_prefetch_factor: 8 +dataloader_num_workers: 8 + +# LoRA +lora_r: 8 +lora_dropout: 0.02 +target_modules: + - down_proj + +# data +train_ds_names: + - pwc + - hotpot_qa + +val_ds_names: + - pwc + - hotpot_qa + +test_ds_names: + - pwc + - hotpot_qa diff --git a/configs/qa.yaml b/configs/qa.yaml new file mode 100644 index 0000000..a146ade --- /dev/null +++ b/configs/qa.yaml @@ -0,0 +1,65 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-2-2b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 5 +weight_decay: 0.01 +# +warmup_steps: 100 + +dataloader_prefetch_factor: 8 +dataloader_num_workers: 8 +# LoRA +lora_r: 8 +lora_dropout: 0.0 +target_modules: + - down_proj +# data +train_ds_names: + - fw_qa_3_small # ~ 20M + - ctx_qa # 300k + - pwc # 240k + - hotpot_qa # 90k + - squad # 90k + - drop # 77k + - narrativeqa # 40k + - quoref # 11k + - ropes # 11k + - synthetic_convqa # 40k + +val_ds_names: + - fw_qa_3_pretrain + - self_gen/google/gemma-2-2b-it/fw_qa_3 + - self_gen/google/gemma-2-2b-it/ctx_qa + - self_gen/google/gemma-2-2b-it/pwc + - self_gen/google/gemma-2-2b-it/hotpot_qa + - self_gen/google/gemma-2-2b-it/squad + - fw_qa_3 + - ctx_qa + - pwc + - hotpot_qa + - squad + +load_best_model_at_end: false +metric_for_best_model: eval_pwc_loss diff --git a/configs/self_gen_3_and_pretrain_small.yaml b/configs/self_gen_3_and_pretrain_small.yaml new file mode 100644 index 0000000..96fbfcb --- /dev/null +++ b/configs/self_gen_3_and_pretrain_small.yaml @@ -0,0 +1,66 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-2-2b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 5 +weight_decay: 0.01 +# +warmup_steps: 100 + +dataloader_prefetch_factor: 8 +dataloader_num_workers: 8 +# LoRA +lora_r: 8 +lora_dropout: 0.0 +target_modules: + - down_proj +# data +train_ds_names: + - fw_qa_3_small_pretrain # ~20M + - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M + - self_gen/google/gemma-2-2b-it/ctx_qa # 300k + - self_gen/google/gemma-2-2b-it/pwc # 240k + - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k + - self_gen/google/gemma-2-2b-it/squad # 90k + - self_gen/google/gemma-2-2b-it/drop # 77k + - self_gen/google/gemma-2-2b-it/narrativeqa # 40k + - self_gen/google/gemma-2-2b-it/quoref # 11k + - self_gen/google/gemma-2-2b-it/ropes # 11k + - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k + +val_ds_names: + - fw_qa_3_pretrain + - self_gen/google/gemma-2-2b-it/fw_qa_3 + - self_gen/google/gemma-2-2b-it/ctx_qa + - self_gen/google/gemma-2-2b-it/pwc + - self_gen/google/gemma-2-2b-it/hotpot_qa + - self_gen/google/gemma-2-2b-it/squad + - fw_qa_3 + - ctx_qa + - pwc + - hotpot_qa + - squad + +load_best_model_at_end: false +metric_for_best_model: eval_pwc_loss diff --git a/configs/self_gen_3_small.yaml b/configs/self_gen_3_small.yaml index 122ca92..862e0ab 100644 --- a/configs/self_gen_3_small.yaml +++ b/configs/self_gen_3_small.yaml @@ -49,18 +49,17 @@ train_ds_names: - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k val_ds_names: + - fw_qa_3_pretrain - self_gen/google/gemma-2-2b-it/fw_qa_3 - - self_gen/google/gemma-2-2b-it/fw_qa_xl - self_gen/google/gemma-2-2b-it/ctx_qa - self_gen/google/gemma-2-2b-it/pwc - self_gen/google/gemma-2-2b-it/hotpot_qa - self_gen/google/gemma-2-2b-it/squad - fw_qa_3 - - fw_qa_xl - ctx_qa - pwc - hotpot_qa - squad -load_best_model_at_end: true +load_best_model_at_end: false metric_for_best_model: eval_pwc_loss diff --git a/scripts/gemma_data_exp/gemma_qa.sh b/scripts/gemma_data_exp/gemma_qa.sh new file mode 100644 index 0000000..5e1c96e --- /dev/null +++ b/scripts/gemma_data_exp/gemma_qa.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora_medium +#SBATCH --partition=a3 +#SBATCH --nodes=1 +#SBATCH --gpus=4 +#SBATCH --output=outputs/%x-%j.out +#SBATCH --error=outputs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29561 intx_sft.py configs/qa.yaml \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=4 --per_device_train_batch_size=32 \ +--gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ +--target_modules=down_proj \ +--num_self_attends_per_block=4 --num_latent_factor=1 \ +--lora_r=8 \ +--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ +--neftune_noise_alpha=5 --use_light_weight_lora=False \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True --per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ + diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa.sh b/scripts/gemma_data_exp/gemma_self_gen_qa.sh new file mode 100644 index 0000000..72f57cc --- /dev/null +++ b/scripts/gemma_data_exp/gemma_self_gen_qa.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora_medium +#SBATCH --partition=a3 +#SBATCH --nodes=1 +#SBATCH --gpus=4 +#SBATCH --output=outputs/%x-%j.out +#SBATCH --error=outputs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_small.yaml \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=4 --per_device_train_batch_size=32 \ +--gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ +--target_modules=down_proj \ +--num_self_attends_per_block=4 --num_latent_factor=1 \ +--lora_r=8 \ +--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ +--neftune_noise_alpha=5 --use_light_weight_lora=False \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True --per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ + diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh new file mode 100644 index 0000000..85589d3 --- /dev/null +++ b/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh @@ -0,0 +1,23 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora_medium +#SBATCH --partition=a3 +#SBATCH --nodes=1 +#SBATCH --gpus=4 +#SBATCH --output=outputs/%x-%j.out +#SBATCH --error=outputs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29563 intx_sft.py configs/self_gen_3_small_and_pretrain.yaml \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=8 \ +--gradient_accumulation_steps=32 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \ +--target_modules=down_proj \ +--num_self_attends_per_block=4 --num_latent_factor=1 \ +--lora_r=8 \ +--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ +--neftune_noise_alpha=5 --use_light_weight_lora=False \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True --per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +