From 63ecb73fdb0ca44f4b113773f0b47a673220a77f Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 8 Jul 2025 06:40:47 +0000 Subject: [PATCH] bigger perceiver scripts --- scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh | 4 ++-- scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh index c674f73..dc9ac29 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh @@ -18,8 +18,8 @@ uv run accelerate launch --main_process_port $port \ --gradient_accumulation_steps=2 \ --per_device_eval_batch_size=64 \ --target_modules=down_proj \ ---num_blocks=4 \ ---num_self_attn_per_block=3 \ +--num_blocks=8 \ +--num_self_attn_per_block=6 \ --n_latent_queries=208 \ --num_pre_head_layers=1 \ --lora_r=8 \ diff --git a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh index 6df5dd4..c6d229a 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh @@ -18,8 +18,8 @@ uv run accelerate launch --main_process_port $port \ --gradient_accumulation_steps=4 \ --per_device_eval_batch_size=32 \ --target_modules=down_proj \ ---num_blocks=4 \ ---num_self_attn_per_block=3 \ +--num_blocks=8 \ +--num_self_attn_per_block=6 \ --ctx_encoder_type=per_layer_activations \ --n_latent_queries=8 \ --num_pre_head_layers=1 \