From 375c50757c2d84f98fc608b39ca1bb2f64551b46 Mon Sep 17 00:00:00 2001 From: 51616 Date: Mon, 30 Jun 2025 16:41:26 +0900 Subject: [PATCH] eval generative only + scripts --- run_eval.py | 20 +++++------ scripts/short_ctx/gemma_qa_short_ctx.sh | 2 +- .../short_ctx/gemma_qa_short_ctx_no_fw_qa.sh | 34 +++++++++++++++++++ .../short_ctx/gemma_qa_short_ctx_self_gen.sh | 14 ++++---- .../gemma_qa_short_ctx_self_gen_no_fw_qa.sh | 34 +++++++++++++++++++ 5 files changed, 86 insertions(+), 18 deletions(-) create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_no_fw_qa.sh create mode 100644 scripts/short_ctx/gemma_qa_short_ctx_self_gen_no_fw_qa.sh diff --git a/run_eval.py b/run_eval.py index 19cb6f8..6581124 100644 --- a/run_eval.py +++ b/run_eval.py @@ -62,16 +62,16 @@ if __name__ == "__main__": eval_batch_size_gen = cli_args.pop("eval_batch_size_gen") eval_batch_size = cli_args.pop("eval_batch_size") - run_eval( - **cli_args, - # cli_args.checkpoint_path, - # cli_args.model_name_or_path, - # cli_args.eval_batch_size, - # args, - # split=cli_args.split, - eval_batch_size=eval_batch_size, - generative=False, - ) + # run_eval( + # **cli_args, + # # cli_args.checkpoint_path, + # # cli_args.model_name_or_path, + # # cli_args.eval_batch_size, + # # args, + # # split=cli_args.split, + # eval_batch_size=eval_batch_size, + # generative=False, + # ) run_eval( **cli_args, # cli_args.checkpoint_path, diff --git a/scripts/short_ctx/gemma_qa_short_ctx.sh b/scripts/short_ctx/gemma_qa_short_ctx.sh index f9be2b7..a9e3af6 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx.sh @@ -9,7 +9,7 @@ uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ --gpu_ids all --main_process_port 29571 intx_sft.py configs/qa_short_ctx.yaml \ --model_name_or_path=google/gemma-2-2b-it \ ---num_train_epochs=5 \ +--num_train_epochs=2 \ --per_device_train_batch_size=-1 \ --gradient_accumulation_steps=2 \ --per_device_eval_batch_size=64 \ diff --git a/scripts/short_ctx/gemma_qa_short_ctx_no_fw_qa.sh b/scripts/short_ctx/gemma_qa_short_ctx_no_fw_qa.sh new file mode 100644 index 0000000..48a3ead --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_no_fw_qa.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29581 intx_sft.py configs/qa_short_ctx_no_fw_qa.yaml \ +--model_name_or_path=google/gemma-2-2b-it \ +--num_train_epochs=2 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=1e-4 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file diff --git a/scripts/short_ctx/gemma_qa_short_ctx_self_gen.sh b/scripts/short_ctx/gemma_qa_short_ctx_self_gen.sh index 6fbb0d6..7eafe48 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_self_gen.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_self_gen.sh @@ -2,16 +2,16 @@ #SBATCH --job-name=ctxlora #SBATCH --nodes=1 #SBATCH --partition=sakura-gpu -#SBATCH --gpus=4 +#SBATCH --gpus=8 #SBATCH --output=slurm_logs/%x-%j.out #SBATCH --error=slurm_logs/%x-%j.out -uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ --gpu_ids all --main_process_port 29572 intx_sft.py configs/qa_short_ctx_self_gen.yaml \ --model_name_or_path=google/gemma-2-2b-it \ ---num_train_epochs=5 \ +--num_train_epochs=2 \ --per_device_train_batch_size=-1 \ ---gradient_accumulation_steps=2 \ +--gradient_accumulation_steps=4 \ --per_device_eval_batch_size=64 \ --target_modules=down_proj \ --num_self_attends_per_block=8 \ @@ -26,9 +26,9 @@ uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gra --add_negative_prompt=False \ --add_repeat_prompt=False \ --use_sequence_packing=True \ ---max_packed_inp_len=16000 \ ---max_packed_ctx_len=32000 \ +--max_packed_inp_len=4096 \ +--max_packed_ctx_len=8192 \ --per_rank_gen=True \ --per_layer_processing=True \ --gen_lora_l1_reg_coef=0.1 \ ---logging_steps=50 \ No newline at end of file +--logging_steps=50 diff --git a/scripts/short_ctx/gemma_qa_short_ctx_self_gen_no_fw_qa.sh b/scripts/short_ctx/gemma_qa_short_ctx_self_gen_no_fw_qa.sh new file mode 100644 index 0000000..5d83d6c --- /dev/null +++ b/scripts/short_ctx/gemma_qa_short_ctx_self_gen_no_fw_qa.sh @@ -0,0 +1,34 @@ +#!/bin/bash +#SBATCH --job-name=ctxlora +#SBATCH --nodes=1 +#SBATCH --partition=sakura-gpu +#SBATCH --gpus=4 +#SBATCH --output=slurm_logs/%x-%j.out +#SBATCH --error=slurm_logs/%x-%j.out + +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29581 intx_sft.py configs/qa_short_ctx_self_gen_no_fw_qa.yaml \ +--model_name_or_path=google/gemma-2-2b-it \ +--num_train_epochs=2 \ +--per_device_train_batch_size=-1 \ +--gradient_accumulation_steps=2 \ +--per_device_eval_batch_size=64 \ +--target_modules=down_proj \ +--num_self_attends_per_block=8 \ +--num_latent_factor=1 \ +--num_pre_head_layers=1 \ +--lora_r=8 \ +--eval_steps=1000 \ +--save_steps=1000 \ +--learning_rate=1e-4 \ +--lora_dropout=0.0 \ +--neftune_noise_alpha=5 \ +--add_negative_prompt=False \ +--add_repeat_prompt=False \ +--use_sequence_packing=True \ +--max_packed_inp_len=16000 \ +--max_packed_ctx_len=32000 \ +--per_rank_gen=True \ +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ +--logging_steps=50 \ No newline at end of file