From 92ba1ca76cb6e68649d84d723d5d8e6f1565c965 Mon Sep 17 00:00:00 2001
From: 51616 <rujikorn.ch@gmail.com>
Date: Tue, 1 Jul 2025 05:52:47 +0000
Subject: [PATCH] tiny data def

---
 data/generate_fw_edu_qa_v2.py        |  2 +-
 data/generate_fw_edu_qa_v2_repeat.py |  2 +-
 data/self_generate_qa.py             |  2 ++
 src/ctx_to_lora/data/definitions.py  | 16 ++++++++--------
 4 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/data/generate_fw_edu_qa_v2.py b/data/generate_fw_edu_qa_v2.py
index 83d1404..d3a0505 100644
--- a/data/generate_fw_edu_qa_v2.py
+++ b/data/generate_fw_edu_qa_v2.py
@@ -8,7 +8,7 @@ from datasets import Dataset, load_dataset
 from vllm import LLM, SamplingParams
 
 STOP_STRINGS = {
-    "google/gemma-2-2b-it": ["<eos>", "<end_of_turn>"],
+    "google/gemma-3-12b-it": ["<eos>", "<end_of_turn>"],
 }
 
 SYSTEM_TEMPLATE = (
diff --git a/data/generate_fw_edu_qa_v2_repeat.py b/data/generate_fw_edu_qa_v2_repeat.py
index 76e13dd..6c70103 100644
--- a/data/generate_fw_edu_qa_v2_repeat.py
+++ b/data/generate_fw_edu_qa_v2_repeat.py
@@ -8,7 +8,7 @@ from datasets import load_dataset
 from vllm import LLM, SamplingParams
 
 STOP_STRINGS = {
-    "google/gemma-2-2b-it": ["<eos>", "<end_of_turn>"],
+    "google/gemma-3-12b-it": ["<eos>", "<end_of_turn>"],
 }
 
 SYSTEM_TEMPLATE = (
diff --git a/data/self_generate_qa.py b/data/self_generate_qa.py
index dade76e..6e6e7ab 100644
--- a/data/self_generate_qa.py
+++ b/data/self_generate_qa.py
@@ -138,6 +138,8 @@ def self_generate(
     temp = 0.0
     if "_temp_" in ds_name:
         temp = float(ds_name.split("_temp_")[-1].split("/")[0])
+    print(f"Processing dataset: {ds_name}, split: {split}")
+    print(f"Using temperature: {temp}")
     kwargs = dict(
         ds_name=ds_name,
         split=split,
diff --git a/src/ctx_to_lora/data/definitions.py b/src/ctx_to_lora/data/definitions.py
index 96d9341..d6907c1 100644
--- a/src/ctx_to_lora/data/definitions.py
+++ b/src/ctx_to_lora/data/definitions.py
@@ -150,9 +150,9 @@ DS_KWARGS = {
         train=dict(
             path="parquet",
             data_files=glob(
-                "data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_0.parquet"
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_0.parquet"
             ),
-            split="train[:400000]",
+            split="train",
         ),
     ),
     "fw_qa_v2_2k_len_level_0": dict(
@@ -187,9 +187,9 @@ DS_KWARGS = {
         train=dict(
             path="parquet",
             data_files=glob(
-                "data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_1.parquet"
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_1.parquet"
             ),
-            split="train[:200000]",
+            split="train",
         ),
     ),
     "fw_qa_v2_2k_len_level_2": dict(
@@ -210,9 +210,9 @@ DS_KWARGS = {
         train=dict(
             path="parquet",
             data_files=glob(
-                "data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_2.parquet"
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_2.parquet"
             ),
-            split="train[:100000]",
+            split="train",
         ),
     ),
     "fw_qa_v2_2k_len_level_3": dict(
@@ -233,9 +233,9 @@ DS_KWARGS = {
         train=dict(
             path="parquet",
             data_files=glob(
-                "data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_3.parquet"
+                "data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_3.parquet"
             ),
-            split="train[:50000]",
+            split="train",
         ),
     ),
     # "fw_qa_3_small": dict(