new self-gen exp (not using self-gen for qa ds with perfect label + small exp + remove "refusal" prompt)

README.md

@@ -80,7 +80,8 @@ uv run data/download_self_gen_qa.py
 
 Loading self-generated data
 ```python
-from ctx_to_lora.data.processing import load_and_process_dataset
+from ctx_to_lora.model_loading import get_tokenizer
+from ctx_to_lora.data.processing import load_and_process_dataset, get_tokenized_dataset
 
 base_model_name = "google/gemma-2-2b-it"
 

configs/small_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml

@@ -0,0 +1,53 @@
+output_dir: "" # just a placeholder
+bf16: true
+model_name_or_path: google/gemma-3-1b-it
+label_names: ["labels"]
+# eval_on_start: True
+# eval_strategy: "steps"
+# eval_steps: 500
+# save_strategy: "no"
+# # save_steps: 500
+# logging_strategy: "steps"
+# logging_steps: 100
+# use_liger_kernel: true
+# remove_unused_columns: false
+
+# needed to avoid OOM by compute the metrics batch by batch
+# w/o this the trainer stores logits of all sample in memory...
+# batch_eval_metrics: true
+
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+max_val_samples_per_ds: 1000
+# optim: schedule_free_adamw
+
+learning_rate: 0.00004
+# lr_scheduler_type: "constant_with_warmup"
+neftune_noise_alpha: 1
+weight_decay: 0.01
+
+warmup_steps: 100
+
+dataloader_prefetch_factor: 16
+dataloader_num_workers: 8
+
+# LoRA
+lora_r: 8
+lora_dropout: 0.0
+target_modules:
+  - down_proj
+
+# data
+train_ds_names:
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2_2k_len_level_1_small
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
+  # these provide exact tokens needed, no need to use self-gen data
+  - squad_compact
+  - ropes_compact
+  - drop_compact
+
+val_ds_names:
+  - squad
+  - pwc
+  - drop
+  - ropes

configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml

@@ -0,0 +1,53 @@
+output_dir: "" # just a placeholder
+bf16: true
+model_name_or_path: google/gemma-3-1b-it
+label_names: ["labels"]
+# eval_on_start: True
+# eval_strategy: "steps"
+# eval_steps: 500
+# save_strategy: "no"
+# # save_steps: 500
+# logging_strategy: "steps"
+# logging_steps: 100
+# use_liger_kernel: true
+# remove_unused_columns: false
+
+# needed to avoid OOM by compute the metrics batch by batch
+# w/o this the trainer stores logits of all sample in memory...
+# batch_eval_metrics: true
+
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+max_val_samples_per_ds: 1000
+# optim: schedule_free_adamw
+
+learning_rate: 0.00004
+# lr_scheduler_type: "constant_with_warmup"
+neftune_noise_alpha: 1
+weight_decay: 0.01
+
+warmup_steps: 100
+
+dataloader_prefetch_factor: 16
+dataloader_num_workers: 8
+
+# LoRA
+lora_r: 8
+lora_dropout: 0.0
+target_modules:
+  - down_proj
+
+# data
+train_ds_names:
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.5/fw_qa_v2_2k_len_level_3_small
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
+  # these provide exact tokens needed, no need to use self-gen data
+  - squad_compact
+  - ropes_compact
+  - drop_compact
+
+val_ds_names:
+  - squad
+  - pwc
+  - drop
+  - ropes

configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml

@@ -0,0 +1,53 @@
+output_dir: "" # just a placeholder
+bf16: true
+model_name_or_path: google/gemma-3-1b-it
+label_names: ["labels"]
+# eval_on_start: True
+# eval_strategy: "steps"
+# eval_steps: 500
+# save_strategy: "no"
+# # save_steps: 500
+# logging_strategy: "steps"
+# logging_steps: 100
+# use_liger_kernel: true
+# remove_unused_columns: false
+
+# needed to avoid OOM by compute the metrics batch by batch
+# w/o this the trainer stores logits of all sample in memory...
+# batch_eval_metrics: true
+
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+max_val_samples_per_ds: 1000
+# optim: schedule_free_adamw
+
+learning_rate: 0.00004
+# lr_scheduler_type: "constant_with_warmup"
+neftune_noise_alpha: 1
+weight_decay: 0.01
+
+warmup_steps: 100
+
+dataloader_prefetch_factor: 16
+dataloader_num_workers: 8
+
+# LoRA
+lora_r: 8
+lora_dropout: 0.0
+target_modules:
+  - down_proj
+
+# data
+train_ds_names:
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2_2k_len_level_3_small
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
+  # these provide exact tokens needed, no need to use self-gen data
+  - squad_compact
+  - ropes_compact
+  - drop_compact
+
+val_ds_names:
+  - squad
+  - pwc
+  - drop
+  - ropes

configs/tiny_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_tiny.yaml

@@ -40,10 +40,11 @@ target_modules:
 # data
 train_ds_names:
   - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2_2k_len_level_1_tiny
-  - self_gen/google/gemma-2-2b-it_temp_0.0/squad_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/pwc_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/ropes_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/drop_compact
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
+  # these provide exact tokens needed, no need to use self-gen data
+  - squad_compact
+  - ropes_compact
+  - drop_compact
 
 val_ds_names:
   - squad

configs/tiny_exp/qa_short_ctx_self_gen_lv1_tiny.yaml

@@ -39,11 +39,11 @@ target_modules:
 
 # data
 train_ds_names:
-  - self_gen/google/gemma-2-2b-it_temp_0.0/fw_qa_v2_2k_len_level_1_tiny
-  - self_gen/google/gemma-2-2b-it_temp_0.0/squad_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/pwc_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/ropes_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/drop_compact
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/fw_qa_v2_2k_len_level_1_tiny
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
+  - squad_compact
+  - ropes_compact
+  - drop_compact
 
 val_ds_names:
   - squad

configs/tiny_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_tiny.yaml

@@ -40,10 +40,10 @@ target_modules:
 # data
 train_ds_names:
   - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2_2k_len_level_3_tiny
-  - self_gen/google/gemma-2-2b-it_temp_0.0/squad_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/pwc_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/ropes_compact
-  - self_gen/google/gemma-2-2b-it_temp_0.0/drop_compact
+  - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
+  - squad_compact
+  - ropes_compact
+  - drop_compact
 
 val_ds_names:
   - squad

data/self_generate_qa.py

@@ -32,8 +32,8 @@ SYSTEM_TEMPLATE = (
     "### SYSTEM INSTRUCTION ###\n"
     "You are an honest and helpful assistant.\n"
     "You must use the information provided in the context for responding to the question.\n"
-    "If the context does not contain enough information to answer the question, you must say so.\n"
-    "If that is the case, you can answer based on your knowledge, but you must clearly state that you are doing so.\n"
+    # "If the context does not contain enough information to answer the question, you must say so.\n"
+    # "If that is the case, you can answer based on your knowledge, but you must clearly state that you are doing so.\n"
     "**DO NOT** hallucinate or make up information.\n"
     "### END OF SYSTEM INSTRUCTION ###"
 )

src/ctx_to_lora/data/definitions.py

@@ -115,9 +115,21 @@ DS_KWARGS = {
     #     validation=dict(path="sggetao/PwC", split="train[:900]"),
     # ),
     "squad": dict(
-        train=dict(path="rajpurkar/squad", split="train[900:]"),
-        validation=dict(path="rajpurkar/squad", split="train[:900]"),
-        test=dict(path="rajpurkar/squad", split="validation"),
+        train=dict(
+            path="/home/rujikorn_sakana_ai/.cache/huggingface/hub/datasets--rajpurkar--squad",
+            split="train[900:]",
+        ),
+        validation=dict(
+            path="/home/rujikorn_sakana_ai/.cache/huggingface/hub/datasets--rajpurkar--squad",
+            split="train[:900]",
+        ),
+        test=dict(
+            path="/home/rujikorn_sakana_ai/.cache/huggingface/hub/datasets--rajpurkar--squad",
+            split="validation",
+        ),
+        # train=dict(path="rajpurkar/squad", split="train[900:]"),
+        # validation=dict(path="rajpurkar/squad", split="train[:900]"),
+        # test=dict(path="rajpurkar/squad", split="validation"),
     ),
     "squad_compact": dict(
         train=dict(
@@ -155,6 +167,15 @@ DS_KWARGS = {
             split="train",
         ),
     ),
+    "fw_qa_v2_2k_len_level_0_small": dict(
+        train=dict(
+            path="parquet",
+            data_files=glob(
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/00[02468]*level_0.parquet"
+            ),
+            split="train",
+        ),
+    ),
     "fw_qa_v2_2k_len_level_0": dict(
         train=dict(
             path="parquet",
@@ -192,6 +213,15 @@ DS_KWARGS = {
             split="train",
         ),
     ),
+    "fw_qa_v2_2k_len_level_1_small": dict(
+        train=dict(
+            path="parquet",
+            data_files=glob(
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/00[02468]*level_1.parquet"
+            ),
+            split="train",
+        ),
+    ),
     "fw_qa_v2_2k_len_level_2": dict(
         train=dict(
             path="parquet",
@@ -215,6 +245,15 @@ DS_KWARGS = {
             split="train",
         ),
     ),
+    "fw_qa_v2_2k_len_level_2_small": dict(
+        train=dict(
+            path="parquet",
+            data_files=glob(
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/00[02468]*level_2.parquet"
+            ),
+            split="train",
+        ),
+    ),
     "fw_qa_v2_2k_len_level_3": dict(
         train=dict(
             path="parquet",
@@ -238,6 +277,15 @@ DS_KWARGS = {
             split="train",
         ),
     ),
+    "fw_qa_v2_2k_len_level_3_small": dict(
+        train=dict(
+            path="parquet",
+            data_files=glob(
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/00[02468]*level_3.parquet"
+            ),
+            split="train",
+        ),
+    ),
     # "fw_qa_3_small": dict(
     #     train=dict(
     #         path="parquet",