From 7d273f44284e4382dbe964c0d7063cbcc6297655 Mon Sep 17 00:00:00 2001
From: 51616 <rujikorn.ch@gmail.com>
Date: Mon, 6 Jan 2025 13:44:14 +0000
Subject: [PATCH] use adamw_torch_fused and 0.1 hypernet layers dropout

---
 hyperlora/configs.py        | 4 ++++
 hyperlora/modeling_utils.py | 6 +++---
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/hyperlora/configs.py b/hyperlora/configs.py
index 52a57cb..0d80531 100644
--- a/hyperlora/configs.py
+++ b/hyperlora/configs.py
@@ -121,6 +121,10 @@ class ExperimentSetup(str, Enum):
 
 @dataclass
 class TrainingArguments(TrainingArguments):
+    optim: str = field(
+        default="adamw_torch_fused",
+        metadata={"help": "Optimizer."},
+    )
     eval_on_start: bool = field(
         default=True,
         metadata={"help": "Whether to evaluate on the start of training."},
diff --git a/hyperlora/modeling_utils.py b/hyperlora/modeling_utils.py
index 89706ee..9575201 100644
--- a/hyperlora/modeling_utils.py
+++ b/hyperlora/modeling_utils.py
@@ -186,15 +186,15 @@ class MLPResidualBlock(nn.Module):
         if pre_layer_norm:
             layers.append(nn.LayerNorm(input_size))
         layers += [
-            nn.Dropout(0.05),
+            nn.Dropout(0.1),
             nn.Linear(input_size, hidden_size),
             nn.SiLU(),
-            nn.Dropout(0.05),
+            nn.Dropout(0.1),
             nn.Linear(hidden_size, output_size),
             nn.SiLU(),
         ]
         if post_dropout:
-            layers.append(nn.Dropout(0.05))
+            layers.append(nn.Dropout(0.1))
         self.mlp = nn.Sequential(*layers)
 
     def forward(self, x):