diff --git a/hyperlora/configs.py b/hyperlora/configs.py
index 52a57cb..0d80531 100644
--- a/hyperlora/configs.py
+++ b/hyperlora/configs.py
@@ -121,6 +121,10 @@ class ExperimentSetup(str, Enum):
 
 @dataclass
 class TrainingArguments(TrainingArguments):
+    optim: str = field(
+        default="adamw_torch_fused",
+        metadata={"help": "Optimizer."},
+    )
     eval_on_start: bool = field(
         default=True,
         metadata={"help": "Whether to evaluate on the start of training."},
diff --git a/hyperlora/modeling_utils.py b/hyperlora/modeling_utils.py
index 89706ee..9575201 100644
--- a/hyperlora/modeling_utils.py
+++ b/hyperlora/modeling_utils.py
@@ -186,15 +186,15 @@ class MLPResidualBlock(nn.Module):
         if pre_layer_norm:
             layers.append(nn.LayerNorm(input_size))
         layers += [
-            nn.Dropout(0.05),
+            nn.Dropout(0.1),
             nn.Linear(input_size, hidden_size),
             nn.SiLU(),
-            nn.Dropout(0.05),
+            nn.Dropout(0.1),
             nn.Linear(hidden_size, output_size),
             nn.SiLU(),
         ]
         if post_dropout:
-            layers.append(nn.Dropout(0.05))
+            layers.append(nn.Dropout(0.1))
         self.mlp = nn.Sequential(*layers)
 
     def forward(self, x):