magic alpha!

src/ctx_to_lora/model_loading.py

@@ -174,7 +174,7 @@ def get_lora_config(model_dir, **kwargs):
         base_model_name_or_path=model_dir,
         task_type="CAUSAL_LM",
         lora_dropout=kwargs.get("lora_dropout", 0.0),
-        lora_alpha=1,  # 2 / r**0.5,
+        lora_alpha=r ** (3 / 2) * 2,  # 1,  # 2 / r**0.5,
     )
 
     peft_conf_kwargs.update(kwargs)

src/ctx_to_lora/modeling/hypernet.py

@@ -650,9 +650,8 @@ class HyperLoRA(nn.Module):
         flat_loras = None
         if self.target_modules:
             lora_emb = self.layers(lora_emb)
-            d = lora_emb.shape[-1]
             norm = torch.norm(lora_emb, dim=-1, keepdim=True)
-            norm_lora_emb = lora_emb / norm * sqrt(d)  # is this too big??
+            norm_lora_emb = lora_emb / norm
             flat_loras = self.head(norm_lora_emb)
 
         flat_layernorms = None
@@ -801,7 +800,7 @@ class ModulatedPretrainedModel(nn.Module):
             nn.init.normal_(
                 self.hypernet.head.weight,
                 mean=0,
-                std=1
+                std=0.5
                 / sqrt(self.hypernet.config.latent_size + self.hypernet.d_lora * r),
                 # the head outputs per rank lora --> divide by r to scale down grad
             )