apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

training working!: l2norm before head

Browse source
This commit is contained in:
51616 2025-05-05 09:19:10 +00:00
parent cd28af25c0
commit c5cae72d1d
1 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

5
src/ctx_to_lora/modeling_utils.py
View file

@ -776,8 +776,6 @@ class HyperLoRA(nn.Module):
]
else:
layers = [
# nn.Linear(self.d_latent, self.d_latent),
# nn.LayerNorm(self.d_latent),
MLPResidualBlock(
input_size=self.config.latent_size,
hidden_size=self.config.latent_size * 4,
@ -1051,7 +1049,8 @@ class HyperLoRA(nn.Module):
flat_loras = None
if self.target_modules:
lora_emb = self.layers(lora_emb)
flat_loras = self.head(lora_emb)
norm_lora_emb = lora_emb / torch.norm(lora_emb, dim=-1, keepdim=True)
flat_loras = self.head(norm_lora_emb)
flat_layernorms = None
if self.num_extra_modules: