LoRA separate bias param + merger + toy ds (#15)

* separate bias + head weight init

* smaller bias init + sum with 1/sqrt(combined_r) scaling + better  lora merge logic

* ok init

* chunked ctx magic num working

* configs

* no transpose + only sum aggregation

* fix cli

* cli

* 32-256 data

* example cli + smaller dataset

* toy ctx magic num ds size + quantize + remove head bias

* configs removed

* require lora_bias

* Merge remote-tracking branch 'origin/main' into new-arch

configs/toy_exp/ctx_kv_64_1024_self_gen.yaml

@@ -1,20 +0,0 @@
-# LoRA
-lora_r: 8
-lora_dropout: 0.0
-target_modules:
-  - down_proj
-
-# data
-train_ds_names:
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_64_128
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_128_256
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_256_512
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_512_768
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_768_1024
-
-val_ds_names:
-- ctx_kv_64_128
-- ctx_kv_128_256
-- ctx_kv_256_512
-- ctx_kv_512_768
-- ctx_kv_768_1024

configs/toy_exp/ctx_kv_64_128_self_gen.yaml

@@ -1,12 +0,0 @@
-# LoRA
-lora_r: 8
-lora_dropout: 0.0
-target_modules:
-  - down_proj
-
-# data
-train_ds_names:
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_64_128
-
-val_ds_names:
-- ctx_kv_64_128

configs/toy_exp/ctx_kv_64_256_self_gen.yaml

@@ -1,14 +0,0 @@
-# LoRA
-lora_r: 8
-lora_dropout: 0.0
-target_modules:
-  - down_proj
-
-# data
-train_ds_names:
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_64_128
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_kv_128_256
-
-val_ds_names:
-- ctx_kv_64_128
-- ctx_kv_128_256

configs/toy_exp/ctx_magic_number_32_256.yaml

@@ -0,0 +1,14 @@
+# LoRA
+lora_r: 8
+lora_dropout: 0.0
+target_modules:
+  - down_proj
+
+# data
+train_ds_names:
+- ctx_magic_number_32_128
+- ctx_magic_number_128_256
+
+val_ds_names:
+- ctx_magic_number_32_128
+- ctx_magic_number_128_256

configs/toy_exp/ctx_numbers_64_128_self_gen.yaml

@@ -1,12 +0,0 @@
-# LoRA
-lora_r: 8
-lora_dropout: 0.0
-target_modules:
-  - down_proj
-
-# data
-train_ds_names:
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_64_128
-
-val_ds_names:
-- ctx_numbers_64_128

configs/toy_exp/ctx_numbers_64_2048_self_gen.yaml

@@ -1,28 +0,0 @@
-# LoRA
-lora_r: 8
-lora_dropout: 0.0
-target_modules:
-  - down_proj
-
-# data
-train_ds_names:
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_64_128
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_128_256
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_256_512
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_512_768
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_768_1024
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_1024_1280
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_1280_1536
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_1536_1792
-- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/ctx_numbers_1792_2048
-
-val_ds_names:
-- ctx_numbers_64_128
-- ctx_numbers_128_256
-- ctx_numbers_256_512
-- ctx_numbers_512_768
-- ctx_numbers_768_1024
-- ctx_numbers_1024_1280
-- ctx_numbers_1280_1536
-- ctx_numbers_1536_1792
-- ctx_numbers_1792_2048