diff --git a/accelerate_config.yaml b/accelerate_config.yaml new file mode 100644 index 0000000..e12b665 --- /dev/null +++ b/accelerate_config.yaml @@ -0,0 +1,17 @@ +compute_environment: LOCAL_MACHINE +debug: false +distributed_type: MULTI_GPU +downcast_bf16: 'no' +enable_cpu_affinity: true +gpu_ids: all +machine_rank: 0 +main_training_function: main +mixed_precision: bf16 +num_machines: 1 +num_processes: 8 +rdzv_backend: static +same_network: true +tpu_env: [] +tpu_use_cluster: false +tpu_use_sudo: false +use_cpu: false diff --git a/ds_config.json b/ds_config.json deleted file mode 100644 index 44a4344..0000000 --- a/ds_config.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "bf16": { - "enabled": "auto" - }, - "zero_optimization": { - "stage": 0, - "stage3_gather_16bit_weights_on_model_save": "no", - "offload_optimizer": { - "device": "auto" - }, - "offload_param": { - "device": "auto" - } - }, - "gradient_clipping": "auto", - "train_batch_size": "auto", - "train_micro_batch_size_per_gpu": "auto", - "gradient_accumulation_steps": "auto", - "steps_per_print": 2000000 -} \ No newline at end of file diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh index d23707f..6bc665b 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH --job-name=ctxlora #SBATCH --nodes=1 -#SBATCH --partition=a3 +#SBATCH --partition=sakura-gpu #SBATCH --gpus=4 #SBATCH --output=slurm_logs/%x-%j.out #SBATCH --error=slurm_logs/%x-%j.out @@ -25,5 +25,5 @@ default_args=( # Pass all script arguments to the training command # $1 comes first, then defaults, then remaining arguments (which can override defaults) -uv run accelerate launch --main_process_port $port \ ---num_processes=4 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}" +uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \ +--num_processes=4 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}" diff --git a/scripts/short_ctx/gemma_qa_short_ctx_exp_default_8_gpus.sh b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_8_gpus.sh index 5685bd3..d803b2a 100644 --- a/scripts/short_ctx/gemma_qa_short_ctx_exp_default_8_gpus.sh +++ b/scripts/short_ctx/gemma_qa_short_ctx_exp_default_8_gpus.sh @@ -1,7 +1,7 @@ #!/bin/bash #SBATCH --job-name=ctxlora #SBATCH --nodes=1 -#SBATCH --partition=a3 +#SBATCH --partition=sakura-gpu #SBATCH --gpus=8 #SBATCH --output=slurm_logs/%x-%j.out #SBATCH --error=slurm_logs/%x-%j.out @@ -25,5 +25,5 @@ default_args=( # Pass all script arguments to the training command # $1 comes first, then defaults, then remaining arguments (which can override defaults) -uv run accelerate launch --main_process_port $port \ ---num_processes=8 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}" +uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \ +--num_processes=8 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}"