From d26f246937c3089933ce830a0979ab4274d2abf2 Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 8 Jul 2025 06:47:51 +0000 Subject: [PATCH] shared weights perceiver --- src/ctx_to_lora/modeling/idefics2.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/ctx_to_lora/modeling/idefics2.py b/src/ctx_to_lora/modeling/idefics2.py index b07a508..0c7bde8 100644 --- a/src/ctx_to_lora/modeling/idefics2.py +++ b/src/ctx_to_lora/modeling/idefics2.py @@ -628,10 +628,12 @@ class Idefics2PerceiverResampler(Idefics2PreTrainedModel): ] self.layers = nn.ModuleList(first_x_attn + first_self_attn_block) - for _ in range(1, config.num_blocks): + for layer_idx in range(1, config.num_blocks): # cross-attention at the beginning of each block if self.shared_weights: - x_attn = first_x_attn[0] + if layer_idx == 1: + second_x_attn = Idefics2PerceiverLayer(config, is_cross_attn=True) + x_attn = second_x_attn else: x_attn = Idefics2PerceiverLayer(config, is_cross_attn=True) self.layers.append(x_attn)