use adamw_torch_fused and 0.1 hypernet layers dropout

hyperlora/configs.py

@@ -121,6 +121,10 @@ class ExperimentSetup(str, Enum):
 
 @dataclass
 class TrainingArguments(TrainingArguments):
+    optim: str = field(
+        default="adamw_torch_fused",
+        metadata={"help": "Optimizer."},
+    )
     eval_on_start: bool = field(
         default=True,
         metadata={"help": "Whether to evaluate on the start of training."},

hyperlora/modeling_utils.py

@@ -186,15 +186,15 @@ class MLPResidualBlock(nn.Module):
         if pre_layer_norm:
             layers.append(nn.LayerNorm(input_size))
         layers += [
-            nn.Dropout(0.05),
+            nn.Dropout(0.1),
             nn.Linear(input_size, hidden_size),
             nn.SiLU(),
-            nn.Dropout(0.05),
+            nn.Dropout(0.1),
             nn.Linear(hidden_size, output_size),
             nn.SiLU(),
         ]
         if post_dropout:
-            layers.append(nn.Dropout(0.05))
+            layers.append(nn.Dropout(0.1))
         self.mlp = nn.Sequential(*layers)
 
     def forward(self, x):