From 6c5a954188afe5686119f3cf57a2ff4b010b6172 Mon Sep 17 00:00:00 2001 From: 51616 Date: Mon, 21 Jul 2025 09:05:08 +0000 Subject: [PATCH] short ctx exp scripts --- scripts/short_ctx/gemma_qa_short_ctx.sh | 4 +- .../gemma_qa_short_ctx_exp_default.sh | 3 +- ...rt_ctx_exp_default_max_qas_per_sample_1.sh | 42 +++++++++++++++++++ ...a_qa_short_ctx_exp_default_more_prehead.sh | 40 ++++++++++++++++++ ...fault_more_prehead_max_qas_per_sample_1.sh | 41 ++++++++++++++++++ .../short_ctx/gemma_qa_short_ctx_old_conf.sh | 42 +++++++++++++++++++ .../gemma_qa_short_ctx_old_conf_8_gpus.sh | 42 +++++++++++++++++++ ...mma_qa_short_ctx_old_conf_more_grad_acc.sh | 41 ++++++++++++++++++ .../short_ctx/gemma_qa_short_ctx_perceiver.sh | 3 +- .../gemma_qa_short_ctx_perceiver_l2l.sh | 4 +- ...short_ctx_perceiver_l2l_128_max_qas_len.sh | 41 ++++++++++++++++++ ..._short_ctx_perceiver_l2l_32_max_qas_len.sh | 41 ++++++++++++++++++ ...short_ctx_perceiver_l2l_512_max_qas_len.sh | 41 ++++++++++++++++++ ..._short_ctx_perceiver_l2l_64_max_qas_len.sh | 41 ++++++++++++++++++ ..._ctx_perceiver_l2l_max_qas_per_sample_1.sh | 42 +++++++++++++++++++ ...hort_ctx_perceiver_max_qas_per_sample_1.sh | 41 ++++++++++++++++++ 16 files changed, 503 insertions(+), 6 deletions(-) create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_exp_default_max_qas_per_sample_1.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead_max_qas_per_sample_1.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_old_conf.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_old_conf_8_gpus.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_old_conf_more_grad_acc.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_128_max_qas_len.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_32_max_qas_len.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_512_max_qas_len.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_64_max_qas_len.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_max_qas_per_sample_1.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_perceiver_max_qas_per_sample_1.sh diff --git a/scripts/short_ctx/gemma_qa_short_ctx.sh b/scripts/short_ctx/gemma_qa_short_ctx.sh index a9e3af6..abf2e09 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH --job-name=ctxlora #SBATCH --nodes=1 -#SBATCH --partition=sakura-gpu +#SBATCH --partition=a3 #SBATCH --gpus=4 #SBATCH --output=slurm_logs/%x-%j.out #SBATCH --error=slurm_logs/%x-%j.out @@ -31,4 +31,4 @@ uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gra --per_rank_gen=True \ --per_layer_processing=True \ --gen_lora_l1_reg_coef=0.1 \ ---logging_steps=50 \ No newline at end of file +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh index aea1a32..12b3fd8 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh @@ -15,7 +15,7 @@ uv run accelerate launch --main_process_port $port \ --model_name_or_path=google/gemma-2-2b-it \ --max_steps=12_000 \ --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=2 \ +--gradient_accumulation_steps=5 \ --per_device_eval_batch_size=64 \ --target_modules=down_proj \ --num_blocks=8 \ @@ -23,6 +23,7 @@ uv run accelerate launch --main_process_port $port \ --n_latent_queries=208 \ --num_pre_head_layers=1 \ --lora_r=8 \ +--eval_strategy=no \ --eval_steps=1000 \ --save_steps=1000 \ --learning_rate=4e-5 \ diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default_max_qas_per_sample_1.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_max_qas_per_sample_1.sh new file mode 100644 index 0000000..5a473fe --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_max_qas_per_sample_1.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=5 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=0 \ +--n_latent_queries=208 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_strategy=no \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_per_sample=1 \ +--max_qas_len=2048 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead.sh new file mode 100644 index 0000000..07220ab --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead.sh @@ -0,0 +1,40 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=5 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=0 \ +--n_latent_queries=208 \ +--num_pre_head_layers=4 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=2048 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead_max_qas_per_sample_1.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead_max_qas_per_sample_1.sh new file mode 100644 index 0000000..9bb2563 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead_max_qas_per_sample_1.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=5 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=0 \ +--n_latent_queries=208 \ +--num_pre_head_layers=4 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_per_sample=1 \ +--max_qas_len=2048 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_old_conf.sh b/scripts/short_ctx/gemma_qa_short_ctx_old_conf.sh new file mode 100644 index 0000000..7a42174 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_old_conf.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=8 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=9 \ +--num_self_attn_per_block=0 \ +--n_latent_queries=208 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_strategy=no \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=2048 \ +--max_qas_per_sample=1 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_old_conf_8_gpus.sh b/scripts/short_ctx/gemma_qa_short_ctx_old_conf_8_gpus.sh new file mode 100644 index 0000000..d5f1453 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_old_conf_8_gpus.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=8 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--num_train_epochs=5 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=4 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=0 \ +--n_latent_queries=208 \ +--num_pre_head_layers=4 \ +--lora_r=8 \ +--eval_strategy=no \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=2048 \ +--max_qas_per_sample=1 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_old_conf_more_grad_acc.sh b/scripts/short_ctx/gemma_qa_short_ctx_old_conf_more_grad_acc.sh new file mode 100644 index 0000000..086fee3 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_old_conf_more_grad_acc.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=12_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=16 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=0 \ +--n_latent_queries=208 \ +--num_pre_head_layers=4 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=2048 \ +--max_qas_per_sample=1 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh index 3143954..76617d5 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh @@ -13,7 +13,7 @@ echo "Using port: $port" uv run accelerate launch --main_process_port $port \ --num_processes=4 --gpu_ids all intx_sft.py $1 \ --model_name_or_path=google/gemma-2-2b-it \ ---max_steps=12_000 \ +--max_steps=10_000 \ --per_device_train_batch_size=-1 \ --gradient_accumulation_steps=2 \ --per_device_eval_batch_size=64 \ @@ -23,6 +23,7 @@ uv run accelerate launch --main_process_port $port \ --n_latent_queries=208 \ --num_pre_head_layers=1 \ --lora_r=8 \ +--eval_strategy=no \ --eval_steps=1000 \ --save_steps=1000 \ --learning_rate=4e-5 \ diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh index 9d9f5c4..0540c7f 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh @@ -13,7 +13,7 @@ echo "Using port: $port" uv run accelerate launch --main_process_port $port \ --num_processes=4 --gpu_ids all intx_sft.py $1 \ --model_name_or_path=google/gemma-2-2b-it \ ---max_steps=12_000 \ +--max_steps=10_000 \ --per_device_train_batch_size=-1 \ --gradient_accumulation_steps=4 \ --per_device_eval_batch_size=32 \ @@ -37,5 +37,5 @@ uv run accelerate launch --main_process_port $port \ --max_packed_ctx_len=8192 \ --per_rank_gen=True \ --per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 \ +--gen_lora_l1_reg_coef=0.0 \ --logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_128_max_qas_len.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_128_max_qas_len.sh new file mode 100644 index 0000000..781fefb --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_128_max_qas_len.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=50_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=4 \ +--per_device_eval_batch_size=32 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=3 \ +--ctx_encoder_type=per_layer_activations \ +--n_latent_queries=8 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=128 \ +--max_packed_inp_len=4096 \ +--max_packed_ctx_len=4096 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.0 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_32_max_qas_len.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_32_max_qas_len.sh new file mode 100644 index 0000000..378c286 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_32_max_qas_len.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=50_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=4 \ +--per_device_eval_batch_size=32 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=3 \ +--ctx_encoder_type=per_layer_activations \ +--n_latent_queries=8 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=32 \ +--max_packed_inp_len=4096 \ +--max_packed_ctx_len=4096 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.0 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_512_max_qas_len.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_512_max_qas_len.sh new file mode 100644 index 0000000..bf4c9e1 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_512_max_qas_len.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=50_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=4 \ +--per_device_eval_batch_size=32 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=3 \ +--ctx_encoder_type=per_layer_activations \ +--n_latent_queries=8 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=512 \ +--max_packed_inp_len=4096 \ +--max_packed_ctx_len=4096 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_64_max_qas_len.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_64_max_qas_len.sh new file mode 100644 index 0000000..84fb432 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_64_max_qas_len.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=50_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=4 \ +--per_device_eval_batch_size=32 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=3 \ +--ctx_encoder_type=per_layer_activations \ +--n_latent_queries=8 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=64 \ +--max_packed_inp_len=4096 \ +--max_packed_ctx_len=4096 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.0 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_max_qas_per_sample_1.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_max_qas_per_sample_1.sh new file mode 100644 index 0000000..06b898a --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_max_qas_per_sample_1.sh @@ -0,0 +1,42 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=10_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=4 \ +--per_device_eval_batch_size=32 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=3 \ +--ctx_encoder_type=per_layer_activations \ +--n_latent_queries=8 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=2048 \ +--max_qas_per_sample=1 \ +--max_packed_inp_len=4096 \ +--max_packed_ctx_len=8192 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.0 \ +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_max_qas_per_sample_1.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_max_qas_per_sample_1.sh new file mode 100644 index 0000000..6c7f107 --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_max_qas_per_sample_1.sh @@ -0,0 +1,41 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=a3 +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +port=$((10000 + ($SLURM_JOBID % 50000))) +echo "Using port: $port" + +# --gradient_accumulation_steps=8 --gradient_clipping=1.0 +uv run accelerate launch --main_process_port $port \ +--num_processes=4 --gpu_ids all intx_sft.py $1 \ +--model_name_or_path=google/gemma-2-2b-it \ +--max_steps=10_000 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_blocks=8 \ +--num_self_attn_per_block=3 \ +--n_latent_queries=208 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=4e-5 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_qas_len=2048 \ +--max_qas_per_sample=1 \ +--max_packed_inp_len=8192 \ +--max_packed_ctx_len=16384 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50