apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

shared weights perceiver

Browse source
This commit is contained in:
51616 2025-07-08 06:47:51 +00:00
parent 63ecb73fdb
commit d26f246937
1 changed files with 4 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

6
src/ctx_to_lora/modeling/idefics2.py
View file

@ -628,10 +628,12 @@ class Idefics2PerceiverResampler(Idefics2PreTrainedModel):
]
self.layers = nn.ModuleList(first_x_attn + first_self_attn_block)
for _ in range(1, config.num_blocks):
for layer_idx in range(1, config.num_blocks):
# cross-attention at the beginning of each block
if self.shared_weights:
x_attn = first_x_attn[0]
if layer_idx == 1:
second_x_attn = Idefics2PerceiverLayer(config, is_cross_attn=True)
x_attn = second_x_attn
else:
x_attn = Idefics2PerceiverLayer(config, is_cross_attn=True)
self.layers.append(x_attn)