From 6c43c6a329705af275d8e4291d93cec870993464 Mon Sep 17 00:00:00 2001
From: 51616 <rujikorn.ch@gmail.com>
Date: Wed, 4 Jun 2025 14:07:32 +0000
Subject: [PATCH] fw_qa_3_mini_self_gen scripts + sleep before load

---
 ...rank_fac2_per_layer_4_gpus_gen_lora_reg.sh | 35 +++++++++++++++++++
 ...c2_per_layer_4_gpus_gen_lora_reg_repeat.sh | 35 +++++++++++++++++++
 src/ctx_to_lora/data/processing.py            |  6 ++--
 3 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh
 create mode 100644 scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh

diff --git a/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh b/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh
new file mode 100644
index 0000000..8d2c34c
--- /dev/null
+++ b/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#SBATCH --job-name=ctxlora_medium
+#SBATCH --partition=a3
+#SBATCH --nodes=1
+#SBATCH --gpus=4
+#SBATCH --output=outputs/%x-%j.out
+#SBATCH --error=outputs/%x-%j.out
+
+# module load
+# module load cuda/12.1
+# module load cudnn/8.9.7
+# module load nccl/cuda-12.1/2.18.3
+# module load hpcx/2.20
+
+# export OMP_NUM_THREADS=24
+# export TRITON_CACHE_DIR=/tmp/.triton/
+# . ~/miniconda3/etc/profile.d/conda.sh
+# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
+# eval "$@"
+
+uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \
+--gpu_ids all --main_process_port 29564 intx_sft.py configs/self_gen_3_mini.yaml \
+--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10 --per_device_train_batch_size=24 \
+--gradient_accumulation_steps=8 --per_device_eval_batch_size=24 --exp_setup=hyper_lora --aggregator_type=perceiver \
+--target_modules=down_proj \
+--num_self_attends_per_block=4 --num_latent_factor=1 \
+--lora_r=8 \
+--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
+--neftune_noise_alpha=5 --use_light_weight_lora=False \
+--load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \
+--add_repeat_prompt=False \
+--use_sequence_packing=True --per_rank_gen=True \
+--per_layer_processing=True \
+--gen_lora_l1_reg_coef=0.1 \
+
diff --git a/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh b/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh
new file mode 100644
index 0000000..ba0fc8d
--- /dev/null
+++ b/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#SBATCH --job-name=ctxlora_medium
+#SBATCH --partition=a3
+#SBATCH --nodes=1
+#SBATCH --gpus=4
+#SBATCH --output=outputs/%x-%j.out
+#SBATCH --error=outputs/%x-%j.out
+
+# module load
+# module load cuda/12.1
+# module load cudnn/8.9.7
+# module load nccl/cuda-12.1/2.18.3
+# module load hpcx/2.20
+
+# export OMP_NUM_THREADS=24
+# export TRITON_CACHE_DIR=/tmp/.triton/
+# . ~/miniconda3/etc/profile.d/conda.sh
+# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
+# eval "$@"
+
+uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \
+--gpu_ids all --main_process_port 29560 intx_sft.py configs/self_gen_3_mini.yaml \
+--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10 --per_device_train_batch_size=24 \
+--gradient_accumulation_steps=8 --per_device_eval_batch_size=24 --exp_setup=hyper_lora --aggregator_type=perceiver \
+--target_modules=down_proj \
+--num_self_attends_per_block=4 --num_latent_factor=1 \
+--lora_r=8 \
+--eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \
+--neftune_noise_alpha=5 --use_light_weight_lora=False \
+--load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \
+--add_repeat_prompt=True --repeat_prob=0.1 \
+--use_sequence_packing=True --per_rank_gen=True \
+--per_layer_processing=True \
+--gen_lora_l1_reg_coef=0.1 \
+
diff --git a/src/ctx_to_lora/data/processing.py b/src/ctx_to_lora/data/processing.py
index 8ce1424..1bb0fc9 100644
--- a/src/ctx_to_lora/data/processing.py
+++ b/src/ctx_to_lora/data/processing.py
@@ -2,6 +2,7 @@ import hashlib
 import json
 import logging
 import random
+import time
 from collections.abc import Callable, Iterator
 from glob import glob
 from os import path
@@ -420,8 +421,8 @@ def load_and_process_dataset(
         # load the cached ds
         logger.info(f"Loaded processed dataset from {ds_path}")
         return datasets.load_from_disk(ds_path)
-    else:
-        logger.info(f"Loading dataset {ds_name} with split {split}...")
+
+    logger.info(f"Loading dataset {ds_name} with split {split}...")
     try:
         ds_kwargs = get_ds_kwargs(ds_name, split)
         skip = ds_kwargs.pop("skip", None)
@@ -485,6 +486,7 @@ def load_and_process_dataset(
                 num_proc=num_proc,
             )
         ds.save_to_disk(ds_path, num_proc=num_proc)
+        time.sleep(5)
         # force loading the cached version for newly created ds
         ds = datasets.load_from_disk(ds_path)
     return ds