diff --git a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_1.0.yaml b/configs/qa_short_ctx_self_gen_lv3_tiny_temp_1.0.yaml deleted file mode 100644 index 88bfc19..0000000 --- a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_1.0.yaml +++ /dev/null @@ -1,52 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: google/gemma-3-1b-it -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - -warmup_steps: 100 - -dataloader_prefetch_factor: 16 -dataloader_num_workers: 8 - -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj - -# data -train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_1.0/fw_qa_v2_2k_len_level_3_tiny - - self_gen/google/gemma-2-2b-it_temp_1.0/squad_compact - - self_gen/google/gemma-2-2b-it_temp_1.0/pwc_compact - - self_gen/google/gemma-2-2b-it_temp_1.0/ropes_compact - - self_gen/google/gemma-2-2b-it_temp_1.0/drop_compact - -val_ds_names: - - squad - - pwc - - drop - - ropes diff --git a/configs/fw_qa_v2_level_0_tiny.yaml b/configs/tiny_exp/fw_qa_v2_level_0_tiny.yaml similarity index 97% rename from configs/fw_qa_v2_level_0_tiny.yaml rename to configs/tiny_exp/fw_qa_v2_level_0_tiny.yaml index 051f7f3..d626796 100644 --- a/configs/fw_qa_v2_level_0_tiny.yaml +++ b/configs/tiny_exp/fw_qa_v2_level_0_tiny.yaml @@ -44,3 +44,6 @@ train_ds_names: val_ds_names: - fw_qa_v2_2k_len_level_0 - squad + - drop + - ropes + - pwc diff --git a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.3.yaml b/configs/tiny_exp/fw_qa_v2_level_1_tiny.yaml similarity index 75% rename from configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.3.yaml rename to configs/tiny_exp/fw_qa_v2_level_1_tiny.yaml index e60d8ee..57fdfa0 100644 --- a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.3.yaml +++ b/configs/tiny_exp/fw_qa_v2_level_1_tiny.yaml @@ -39,14 +39,11 @@ target_modules: # data train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_0.3/fw_qa_v2_2k_len_level_3_tiny - - self_gen/google/gemma-2-2b-it_temp_0.3/squad_compact - - self_gen/google/gemma-2-2b-it_temp_0.3/pwc_compact - - self_gen/google/gemma-2-2b-it_temp_0.3/ropes_compact - - self_gen/google/gemma-2-2b-it_temp_0.3/drop_compact + - fw_qa_v2_2k_len_level_1_tiny val_ds_names: + - fw_qa_v2_2k_len_level_0 - squad - - pwc - drop - ropes + - pwc diff --git a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.5.yaml b/configs/tiny_exp/fw_qa_v2_level_2_tiny.yaml similarity index 75% rename from configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.5.yaml rename to configs/tiny_exp/fw_qa_v2_level_2_tiny.yaml index 43e069c..e6390cd 100644 --- a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.5.yaml +++ b/configs/tiny_exp/fw_qa_v2_level_2_tiny.yaml @@ -39,14 +39,11 @@ target_modules: # data train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_0.5/fw_qa_v2_2k_len_level_3_tiny - - self_gen/google/gemma-2-2b-it_temp_0.5/squad_compact - - self_gen/google/gemma-2-2b-it_temp_0.5/pwc_compact - - self_gen/google/gemma-2-2b-it_temp_0.5/ropes_compact - - self_gen/google/gemma-2-2b-it_temp_0.5/drop_compact + - fw_qa_v2_2k_len_level_2_tiny val_ds_names: + - fw_qa_v2_2k_len_level_0 - squad - - pwc - drop - ropes + - pwc diff --git a/configs/fw_qa_v2_level_3_tiny.yaml b/configs/tiny_exp/fw_qa_v2_level_3_tiny.yaml similarity index 97% rename from configs/fw_qa_v2_level_3_tiny.yaml rename to configs/tiny_exp/fw_qa_v2_level_3_tiny.yaml index 9f25503..9d782f3 100644 --- a/configs/fw_qa_v2_level_3_tiny.yaml +++ b/configs/tiny_exp/fw_qa_v2_level_3_tiny.yaml @@ -44,3 +44,6 @@ train_ds_names: val_ds_names: - fw_qa_v2_2k_len_level_0 - squad + - drop + - ropes + - pwc diff --git a/configs/qa_short_ctx_no_fw_qa.yaml b/configs/tiny_exp/qa_short_ctx_no_fw_qa.yaml similarity index 100% rename from configs/qa_short_ctx_no_fw_qa.yaml rename to configs/tiny_exp/qa_short_ctx_no_fw_qa.yaml diff --git a/configs/qa_short_ctx_self_gen_lv0_tiny.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv0_tiny.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv0_tiny.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv0_tiny.yaml diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv1_tiny.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny.yaml diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml diff --git a/configs/qa_short_ctx_self_gen_lv2_tiny.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv2_tiny.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv2_tiny.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv2_tiny.yaml diff --git a/configs/qa_short_ctx_self_gen_lv3_tiny.yaml b/configs/tiny_exp/qa_short_ctx_self_gen_lv3_tiny.yaml similarity index 100% rename from configs/qa_short_ctx_self_gen_lv3_tiny.yaml rename to configs/tiny_exp/qa_short_ctx_self_gen_lv3_tiny.yaml diff --git a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.7.yaml b/configs/tiny_exp/qa_short_ctx_tiny.yaml similarity index 75% rename from configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.7.yaml rename to configs/tiny_exp/qa_short_ctx_tiny.yaml index eecf57d..1467040 100644 --- a/configs/qa_short_ctx_self_gen_lv3_tiny_temp_0.7.yaml +++ b/configs/tiny_exp/qa_short_ctx_tiny.yaml @@ -37,16 +37,23 @@ lora_dropout: 0.0 target_modules: - down_proj -# data +# short ctx data train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_0.7/fw_qa_v2_2k_len_level_3_tiny - - self_gen/google/gemma-2-2b-it_temp_0.7/squad_compact - - self_gen/google/gemma-2-2b-it_temp_0.7/pwc_compact - - self_gen/google/gemma-2-2b-it_temp_0.7/ropes_compact - - self_gen/google/gemma-2-2b-it_temp_0.7/drop_compact + - fw_qa_v2_2k_len_level_3_tiny + - squad_compact + - pwc_compact + - drop_compact + - ropes_compact + +# the contexts are kinda long-ish +# - narrativeqa +# - quoref +# - synthetic_convqa val_ds_names: + - fw_qa_v2_2k_len_level_0 - squad - - pwc - drop - ropes + - pwc + diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_concat_latent_context_attn.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_concat_latent_context_attn.sh new file mode 100644 index 0000000..59d607c --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_concat_latent_context_attn.sh @@ -0,0 +1,40 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--n_cross_attn_layers=8 \ +--concat_latents_context=True \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh new file mode 100644 index 0000000..b78f1ee --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--n_cross_attn_layers=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_self_attn.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_self_attn.sh new file mode 100644 index 0000000..64b46b0 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_self_attn.sh @@ -0,0 +1,39 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--n_cross_attn_layers=1 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/submit_tiny_exp_jobs.sh b/scripts/submit_tiny_exp_jobs.sh new file mode 100755 index 0000000..f530b43 --- /dev/null +++ b/scripts/submit_tiny_exp_jobs.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Script to submit SLURM jobs for all configs in configs/tiny_exp/ +# with random sleep intervals between submissions + +CONFIG_DIR="configs/tiny_exp" +SCRIPT_PATH="scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh" + +# Check if config directory exists +if [ ! -d "$CONFIG_DIR" ]; then + echo "Error: Config directory $CONFIG_DIR does not exist" + exit 1 +fi + +# Check if script exists +if [ ! -f "$SCRIPT_PATH" ]; then + echo "Error: Script $SCRIPT_PATH does not exist" + exit 1 +fi + +# Get all yaml config files +configs=($(find "$CONFIG_DIR" -name "*.yaml" -type f)) + +if [ ${#configs[@]} -eq 0 ]; then + echo "No .yaml config files found in $CONFIG_DIR" + exit 1 +fi + +echo "Found ${#configs[@]} config files:" +for config in "${configs[@]}"; do + echo " - $config" +done + +echo "" +echo "Starting job submissions..." + +# Submit jobs with random sleep intervals +for config in "${configs[@]}"; do + echo "Submitting job for config: $config" + sbatch "$SCRIPT_PATH" "$config" + + # Random sleep between 3-30 seconds + sleep_time=$((3 + RANDOM % 28)) + echo "Waiting $sleep_time seconds before next submission..." + sleep $sleep_time +done + +echo "All jobs submitted!" \ No newline at end of file