Refactor_and_improve_data (#7)

* faster slice

* add facts + ctx_qa

* new configs

* new scripts

* intx_sft.py to train.py

* add kaggle for downloading facts

* max_new_tokens cli for eval

* generate negative_nq

* scripts + configs

* default vals

* max_val_samples_per_ds=500

* more efficient layer-to-layer ctx encoder

* use_per_ctx_average_loss

* faster processing

* small exp distill

* scripts

* more robust watcher

* per-module l1_norm avg

* per-ctx average loss

* clear_gpu

scripts/short_ctx/gemma_fw_qa_level_0.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29562 intx_sft.py configs/fw_qa_v2_level_0.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_fw_qa_level_0_tiny.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29562 intx_sft.py configs/fw_qa_v2_level_0_tiny.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_fw_qa_level_1.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29563 intx_sft.py configs/fw_qa_v2_level_1.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_fw_qa_level_2.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29564 intx_sft.py configs/fw_qa_v2_level_2.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_fw_qa_level_3.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29565 intx_sft.py configs/fw_qa_v2_level_3.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_fw_qa_level_3_tiny.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29563 intx_sft.py configs/fw_qa_v2_level_3_tiny.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29571 intx_sft.py configs/qa_short_ctx.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=2 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_exp_default.sh

@@ -9,33 +9,21 @@
 port=$((10000 + ($SLURM_JOBID % 50000)))
 echo "Using port: $port"
 
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
+# Default arguments - these can be overridden by passing arguments to this script
+default_args=(
+    "--model_name_or_path=google/gemma-2-2b-it"
+    "--max_steps=50_000"
+    "--target_modules=down_proj"
+    "--lora_r=8"
+    "--eval_strategy=no"
+    "--max_qas_len=2048"
+    "--max_qas_per_sample=1"
+    "--per_rank_gen=True"
+    "--per_layer_processing=True"
+    "--gen_lora_l1_reg_coef=0.1"
+)
+
+# Pass all script arguments to the training command
+# $1 comes first, then defaults, then remaining arguments (which can override defaults)
 uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=12_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=5 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50
+--num_processes=4 --gpu_ids all train.py $1 "${default_args[@]}" "${@:2}" 

scripts/short_ctx/gemma_qa_short_ctx_exp_default_max_qas_per_sample_1.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=12_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=5 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_per_sample=1 \
---max_qas_len=2048 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=12_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=5 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=4 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_exp_default_more_prehead_max_qas_per_sample_1.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=12_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=5 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=4 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_per_sample=1 \
---max_qas_len=2048 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_exp_self_attn.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=12_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---n_cross_attn_layers=1 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_no_fw_qa.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29581 intx_sft.py configs/qa_short_ctx_no_fw_qa.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=2 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=1e-4 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_old_conf.sh

@@ -1,40 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((29560 + ($SLURM_JOBID % 10)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=3 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=8 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=9 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=8192 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_old_conf_8_gpus.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=8
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=4 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_old_conf_distill.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((29560 + ($SLURM_JOBID % 10)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=3 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=16 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=9 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=4096 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50 \
---use_kl_loss=True

scripts/short_ctx/gemma_qa_short_ctx_old_conf_l2l_distill.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((29560 + ($SLURM_JOBID % 10)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=3 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=16 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=9 \
---num_self_attn_per_block=0 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=4096 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50 \
---use_kl_loss=True

scripts/short_ctx/gemma_qa_short_ctx_old_conf_more_grad_acc.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=12_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=16 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=0 \
---n_latent_queries=208 \
---num_pre_head_layers=4 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=30_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---n_latent_queries=208 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_strategy=no \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=30_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=8 \
---per_device_eval_batch_size=32 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=8192 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.0 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_128_max_qas_len.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=50_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=32 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=128 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=4096 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.0 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_32_max_qas_len.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=50_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=32 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=32 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=4096 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.0 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_512_max_qas_len.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=50_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=32 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=512 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=4096 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_64_max_qas_len.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=50_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=32 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=64 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=4096 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.0 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_l2l_max_qas_per_sample_1.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=30_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=8 \
---per_device_eval_batch_size=32 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---ctx_encoder_type=per_layer_activations \
---n_latent_queries=8 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=8192 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.0 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_perceiver_max_qas_per_sample_1.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=a3
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-port=$((10000 + ($SLURM_JOBID % 50000)))
-echo "Using port: $port"
-
-# --gradient_accumulation_steps=8 --gradient_clipping=1.0
-uv run accelerate launch --main_process_port $port \
---num_processes=4 --gpu_ids all intx_sft.py $1 \
---model_name_or_path=google/gemma-2-2b-it \
---max_steps=10_000 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_blocks=8 \
---num_self_attn_per_block=3 \
---n_latent_queries=208 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_qas_len=2048 \
---max_qas_per_sample=1 \
---max_packed_inp_len=8192 \
---max_packed_ctx_len=16384 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_self_gen.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=8
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=8 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29572 intx_sft.py configs/qa_short_ctx_self_gen.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=2 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=4 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=4096 \
---max_packed_ctx_len=8192 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_qa_short_ctx_self_gen_no_fw_qa.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29581 intx_sft.py configs/qa_short_ctx_self_gen_no_fw_qa.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=2 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=1e-4 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_squad.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29561 intx_sft.py configs/squad.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50

scripts/short_ctx/gemma_squad_and_fw_qa_level_3.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-#SBATCH --job-name=ctxlora
-#SBATCH --nodes=1
-#SBATCH --partition=sakura-gpu
-#SBATCH --gpus=4
-#SBATCH --output=slurm_logs/%x-%j.out
-#SBATCH --error=slurm_logs/%x-%j.out
-
-uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=2 --gradient_clipping=1.0 \
---gpu_ids all --main_process_port 29566 intx_sft.py configs/squad_fw_qa_v2_level_3.yaml \
---model_name_or_path=google/gemma-2-2b-it \
---num_train_epochs=5 \
---per_device_train_batch_size=-1 \
---gradient_accumulation_steps=2 \
---per_device_eval_batch_size=64 \
---target_modules=down_proj \
---num_self_attends_per_block=8 \
---num_latent_factor=1 \
---num_pre_head_layers=1 \
---lora_r=8 \
---eval_steps=1000 \
---save_steps=1000 \
---learning_rate=4e-5 \
---lora_dropout=0.0 \
---neftune_noise_alpha=5 \
---add_negative_prompt=False \
---add_repeat_prompt=False \
---use_sequence_packing=True \
---max_packed_inp_len=16000 \
---max_packed_ctx_len=32000 \
---per_rank_gen=True \
---per_layer_processing=True \
---gen_lora_l1_reg_coef=0.1 \
---logging_steps=50