sakura scripts

accelerate_config.yaml

@@ -0,0 +1,17 @@
+compute_environment: LOCAL_MACHINE
+debug: false
+distributed_type: MULTI_GPU
+downcast_bf16: 'no'
+enable_cpu_affinity: true
+gpu_ids: all
+machine_rank: 0
+main_training_function: main
+mixed_precision: bf16
+num_machines: 1
+num_processes: 8
+rdzv_backend: static
+same_network: true
+tpu_env: []
+tpu_use_cluster: false
+tpu_use_sudo: false
+use_cpu: false

ds_config.json

@@ -1,20 +0,0 @@
-{
-    "bf16": {
-        "enabled": "auto"
-    },
-    "zero_optimization": {
-        "stage": 0,
-        "stage3_gather_16bit_weights_on_model_save": "no",
-        "offload_optimizer": {
-            "device": "auto"
-        },
-        "offload_param": {
-            "device": "auto"
-        }
-    },
-    "gradient_clipping": "auto",
-    "train_batch_size": "auto",
-    "train_micro_batch_size_per_gpu": "auto",
-    "gradient_accumulation_steps": "auto",
-    "steps_per_print": 2000000
-}

scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #SBATCH --job-name=ctxlora
 #SBATCH --nodes=1
-#SBATCH --partition=a3
+#SBATCH --partition=sakura-gpu
 #SBATCH --gpus=4
 #SBATCH --output=slurm_logs/%x-%j.out
 #SBATCH --error=slurm_logs/%x-%j.out
@@ -25,5 +25,5 @@ default_args=(
 
 # Pass all script arguments to the training command
 # $1 comes first, then defaults, then remaining arguments (which can override defaults)
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}" 
+uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
+--num_processes=4 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}"

scripts/short_ctx/gemma_qa_short_ctx_exp_default_8_gpus.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 #SBATCH --job-name=ctxlora
 #SBATCH --nodes=1
-#SBATCH --partition=a3
+#SBATCH --partition=sakura-gpu
 #SBATCH --gpus=8
 #SBATCH --output=slurm_logs/%x-%j.out
 #SBATCH --error=slurm_logs/%x-%j.out
@@ -25,5 +25,5 @@ default_args=(
 
 # Pass all script arguments to the training command
 # $1 comes first, then defaults, then remaining arguments (which can override defaults)
-uv run accelerate launch --main_process_port $port \
---num_processes=8 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}" 
+uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
+--num_processes=8 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}"