vlm ctx-encoder + slrum scripts + qwen self-gen

scripts/main_exp/self_gen_data_mistral_7b_lvl_1.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name=selfgen
+#SBATCH --time=5-00:00
+#SBATCH --partition=a3
+#SBATCH --ntasks-per-node=1
+#SBATCH --nodes=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/%x-%A_%a.out
+#SBATCH --error=slurm_logs/%x-%A_%a.err
+#SBATCH --cpus-per-task=16
+
+# Collect matching files into an array (bash will expand the glob)
+files=(data/raw_datasets/fw_qa_v2/min_0_to_2000/{000..013}*_level_1.parquet)
+
+# Default to 0 if SLURM_ARRAY_TASK_ID is unset
+idx=${SLURM_ARRAY_TASK_ID:-0}
+
+# Bounds check
+if (( idx < 0 || idx >= ${#files[@]} )); then
+    echo "Error: SLURM_ARRAY_TASK_ID $idx out of range (0..$((${#files[@]}-1)))." >&2
+    exit 1
+fi
+
+selected_file=${files[$idx]}
+
+echo "Processing file index $idx: $selected_file"
+
+uv run data/self_generate_qa.py \
+--vllm_model mistralai/Mistral-7B-Instruct-v0.2 \
+--glob_pattern "$selected_file" \
+--closed_qa_prob 1.0 \
+--max_new_tokens 1024

scripts/main_exp/self_gen_data_qwen_4b_lvl_1.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+#SBATCH --job-name=selfgen
+#SBATCH --time=5-00:00
+#SBATCH --partition=a3
+#SBATCH --ntasks-per-node=1
+#SBATCH --nodes=1
+#SBATCH --gres=gpu:1
+#SBATCH --output=slurm_logs/%x-%A_%a.out
+#SBATCH --error=slurm_logs/%x-%A_%a.err
+#SBATCH --cpus-per-task=16
+
+# Collect matching files into an array (bash will expand the glob)
+files=(data/raw_datasets/fw_qa_v2/min_0_to_2000/{000..013}*_level_1.parquet)
+
+# Default to 0 if SLURM_ARRAY_TASK_ID is unset
+idx=${SLURM_ARRAY_TASK_ID:-0}
+
+# Bounds check
+if (( idx < 0 || idx >= ${#files[@]} )); then
+    echo "Error: SLURM_ARRAY_TASK_ID $idx out of range (0..$((${#files[@]}-1)))." >&2
+    exit 1
+fi
+
+selected_file=${files[$idx]}
+
+echo "Processing file index $idx: $selected_file"
+
+uv run data/self_generate_qa.py \
+--vllm_model Qwen/Qwen3-4B-Instruct-2507 \
+--glob_pattern "$selected_file" \
+--closed_qa_prob 1.0 \
+--max_new_tokens 1024

scripts/main_exp/train-cross-enc-chunk-slurm.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+#SBATCH --job-name=ctxlora
+#SBATCH --nodes=1
+#SBATCH --partition=a3
+#SBATCH --gpus=8
+#SBATCH --output=slurm_logs/%x-%j.out
+#SBATCH --error=slurm_logs/%x-%j.out
+
+port=$((10000 + ($SLURM_JOBID % 50000)))
+echo "Using port: $port"
+
+# port=29051
+
+uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
+--num_processes=8 --gpu_ids all train.py \
+configs/main_exp/self_gen_lv1_closed_qa_1_l2l.yaml \
+--model_name_or_path=google/gemma-2-2b-it \
+--target_modules=down_proj --lora_r=8 \
+--eval_strategy=no --max_qas_len=2048 --max_qas_per_sample=1 \
+--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
+--max_steps=20000 --gradient_accumulation_steps=8 --max_packed_inp_len=4096 \
+--max_packed_ctx_len=4096 --use_per_ctx_average_loss=True --use_kl_loss=True \
+--quantize_ctx_encoder=True --ctx_encoder_model_name_or_path=google/gemma-3-4b-it \
+--max_ctx_chunk_len=512 \
+--min_ctx_chunk_len=25 \
+--num_chunk_probs='{"1":"0.5", "2":"0.125", "3":"0.0625", "4":"0.0625", "5":"0.0625", "6":"0.0625", "7":"0.0625", "8":"0.0625"}' \
+--warmup_steps=2000 \
+--learning_rate=2e-5 \
+"$@"

scripts/main_exp/train-cross-enc-slurm.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --job-name=ctxlora
+#SBATCH --nodes=1
+#SBATCH --partition=a3
+#SBATCH --gpus=8
+#SBATCH --output=slurm_logs/%x-%j.out
+#SBATCH --error=slurm_logs/%x-%j.out
+
+port=$((10000 + ($SLURM_JOBID % 50000)))
+echo "Using port: $port"
+
+# port=29051
+
+uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
+--num_processes=8 --gpu_ids all train.py \
+configs/main_exp/self_gen_lv1_closed_qa_1_l2l.yaml \
+--model_name_or_path=google/gemma-2-2b-it \
+--target_modules=down_proj --lora_r=8 \
+--eval_strategy=no --max_qas_len=2048 --max_qas_per_sample=1 \
+--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
+--max_steps=80000 --gradient_accumulation_steps=8 --max_packed_inp_len=4096 \
+--max_packed_ctx_len=4096 --use_per_ctx_average_loss=True --use_kl_loss=True \
+--quantize_ctx_encoder=True --ctx_encoder_model_name_or_path=google/gemma-3-4b-it \
+"$@"

scripts/main_exp/train-qwen-slurm.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+#SBATCH --job-name=ctxlora
+#SBATCH --nodes=1
+#SBATCH --partition=a3
+#SBATCH --gpus=8
+#SBATCH --output=slurm_logs/%x-%j.out
+#SBATCH --error=slurm_logs/%x-%j.out
+
+port=$((10000 + ($SLURM_JOBID % 50000)))
+echo "Using port: $port"
+
+# port=29051
+
+uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
+--num_processes=8 --gpu_ids all train.py \
+configs/main_exp/qwen/self_gen_lv1_closed_qa_1_l2l.yaml \
+--model_name_or_path=Qwen/Qwen3-4B-Instruct-2507 \
+--target_modules=down_proj --lora_r=8 \
+--eval_strategy=no --max_qas_len=1024 --max_qas_per_sample=1 \
+--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
+--max_steps=80000 --gradient_accumulation_steps=16 --max_packed_inp_len=2048 \
+--max_packed_ctx_len=2048 --use_per_ctx_average_loss=True --use_kl_loss=True \
+--quantize_ctx_encoder=True \
+"$@"

scripts/main_exp/train-qwen.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+port=29051
+
+uv run accelerate launch --config_file accelerate_config.yaml --main_process_port $port \
+--num_processes=8 --gpu_ids all train.py \
+configs/main_exp/qwen/self_gen_lv1_closed_qa_1_l2l.yaml \
+--model_name_or_path=Qwen/Qwen3-4B-Instruct-2507 \
+--target_modules=down_proj --lora_r=8 \
+--eval_strategy=no --max_qas_len=2048 --max_qas_per_sample=1 \
+--per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 \
+--max_steps=80000 --gradient_accumulation_steps=8 --max_packed_inp_len=4096 \
+--max_packed_ctx_len=4096 --use_per_ctx_average_loss=True --use_kl_loss=True \
+--quantize_ctx_encoder=True