From 4a2960cd64f0de793e7a26072a7401712336d471 Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 10 Jun 2025 15:40:19 +0000 Subject: [PATCH] gemma data exp scripts --- scripts/gemma_data_exp/gemma_self_gen_qa.sh | 6 ++--- .../gemma_self_gen_qa_and_pretrain.sh | 4 ++-- .../gemma_data_exp/gemma_self_gen_qa_mini.sh | 24 +++++++++++++++++++ 3 files changed, 29 insertions(+), 5 deletions(-) create mode 100644 scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa.sh b/scripts/gemma_data_exp/gemma_self_gen_qa.sh index 72f57cc..3dced98 100644 --- a/scripts/gemma_data_exp/gemma_self_gen_qa.sh +++ b/scripts/gemma_data_exp/gemma_self_gen_qa.sh @@ -6,10 +6,10 @@ #SBATCH --output=outputs/%x-%j.out #SBATCH --error=outputs/%x-%j.out -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ --gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=4 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=3 --per_device_train_batch_size=8 \ +--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ --target_modules=down_proj \ --num_self_attends_per_block=4 --num_latent_factor=1 \ --lora_r=8 \ diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh index f8319e8..1847ad3 100644 --- a/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh +++ b/scripts/gemma_data_exp/gemma_self_gen_qa_and_pretrain.sh @@ -8,8 +8,8 @@ uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \ --gpu_ids all --main_process_port 29563 intx_sft.py configs/self_gen_3_and_pretrain_small.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=8 \ ---gradient_accumulation_steps=32 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=4 \ +--gradient_accumulation_steps=32 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ --target_modules=down_proj \ --num_self_attends_per_block=4 --num_latent_factor=1 \ --lora_r=8 \ diff --git a/scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh b/scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh new file mode 100644 index 0000000..faf9ea5 --- /dev/null +++ b/scripts/gemma_data_exp/gemma_self_gen_qa_mini.sh @@ -0,0 +1,24 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora_medium +#SBATCH --partition=a3 +#SBATCH --nodes=1 +#SBATCH --gpus=4 +#SBATCH --output=outputs/%x-%j.out +#SBATCH --error=outputs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29562 intx_sft.py configs/self_gen_3_mini.yaml \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=3 --per_device_train_batch_size=8 \ +--gradient_accumulation_steps=16 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 --num_latent_factor=1 \ +--lora_r=8 \ +--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ +--neftune_noise_alpha=5 --use_light_weight_lora=False \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True --max_packed_inp_len=16384 --max_packed_ctx_len=32768 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +