remove dropout

batch_gemma_llama_vision.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-#SBATCH --job-name=gemma_pixtral
+#SBATCH --job-name=gemma_llama_vision
 #SBATCH --partition=a3
 #SBATCH --nodes=1
 #SBATCH --gpus=8
@@ -22,7 +22,7 @@ accelerate launch --num_processes=8 --gradient_accumulation_steps=4 --gradient_c
 --gpu_ids all --main_process_port 29554 intx_sft.py configs/pretrain_all.yaml \
 --model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=4 \
 --gradient_accumulation_steps=4 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \
---target_modules=down_proj,up_proj --num_blocks=1 --num_self_attends_per_block=16 \
+--target_modules=down_proj --num_blocks=1 --num_self_attends_per_block=16 \
 --self_attention_widening_factor=1 --eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 \
 --neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \
 --load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \

src/ctx_to_lora/configs.py

@@ -245,7 +245,7 @@ class LoRAArguments:
         metadata={"help": ("LoRA R value.")},
     )
     lora_dropout: Optional[float] = field(
-        default=0.05,
+        default=0,
         metadata={"help": ("LoRA dropout.")},
     )
     target_modules: Optional[list[str]] = field(
@@ -340,6 +340,10 @@ class HypernetArguments:
         default=128,
         metadata={"help": "Latent size for light-weight LoRA."},
     )
+    dropout_rate: float = field(
+        default=0.0,
+        metadata={"help": "Dropout rate for HyperLoRA."},
+    )
     # trainable_base_modules: Optional[list[str]] = field(
     #     default=None,
     #     metadata={"help": ("Modules to train of the base model.")},

src/ctx_to_lora/model_loading.py

@@ -180,7 +180,7 @@ def get_lora_config(model_dir, **kwargs):
         peft_type=PeftType.LORA,
         base_model_name_or_path=model_dir,
         task_type="CAUSAL_LM",
-        lora_dropout=kwargs.get("lora_dropout", 0.05),
+        lora_dropout=kwargs.get("lora_dropout", 0.0),
         lora_alpha=r ** (3 / 2) * 2,
     )
 

src/ctx_to_lora/modeling_utils.py

@@ -103,6 +103,7 @@ class HypernetConfig:
     latent_size: int
     use_light_weight_lora: bool
     light_weight_latent_size: int
+    dropout_rate: float
 
     lora_config: LoraConfig
     module_names: dict[str, list[str]]
@@ -215,21 +216,22 @@ class MLPResidualBlock(nn.Module):
         output_size: int,
         pre_layer_norm: bool = True,
         post_dropout: bool = True,
+        dropout_rate: float = 0,
     ):
         super().__init__()
         layers = []
         if pre_layer_norm:
             layers.append(nn.LayerNorm(input_size))
         layers += [
-            nn.Dropout(0.05),
+            nn.Dropout(dropout_rate),
             nn.Linear(input_size, hidden_size),
             nn.SiLU(),
-            nn.Dropout(0.05),
+            nn.Dropout(dropout_rate),
             nn.Linear(hidden_size, output_size),
             nn.SiLU(),
         ]
         if post_dropout:
-            layers.append(nn.Dropout(0.05))
+            layers.append(nn.Dropout(dropout_rate))
         self.mlp = nn.Sequential(*layers)
 
     def forward(self, x):
@@ -437,6 +439,7 @@ class HyperLoRA(nn.Module):
             input_size=self.config.latent_size,
             hidden_size=self.config.latent_size * 4,
             output_size=self.config.latent_size,
+            dropout_rate=self.config.dropout_rate,
         )
 
         if self.config.use_light_weight_lora: