hf data

README.md

@@ -72,11 +72,38 @@ uv run python data/generate_fav_num_big.py
 ```
 
 ### Data Generation (v2)
-Recursively generate more data!
+***Download a subset of generated Fineweb-QA directly***
+```bash
+uv run data/download_generated_fineweb_qa.py
+```
+
+***Generate data from scratch***
+0. download fineweb_edu to `data/raw_datasets/fineweb_edu
+```bash
+# [WIP] there are other datasets where we used openAI to gen QA pairs
+
+# 0. download fineweb_edu to `data/raw_datasets/fineweb_edu
+uv run data/download_fineweb_edu.py
+```
+
+1. Recursively generate more data! (depends on step 0)
 ```bash
 # run from 000 to 0013
 run uv run python data/generate_fw_edu_qa_v2.py --shard_pattern "000_00000" --n_qa_pairs=5 --vllm_model=google/gemma-3-12b-it --max_length=2000 --max_model_length=2048; run uv run python data/generate_fw_edu_qa_v2_repeat.py --shard_pattern "min_0_to_2000/000*level_0*" --n_qa_pairs=5 --vllm_model=google/gemma-3-12b-it; run uv run python data/generate_fw_edu_qa_v2_repeat.py --shard_pattern "min_0_to_2000/000*level_1*" --n_qa_pairs=5 --vllm_model=google/gemma-3-12b-it; run uv run python data/generate_fw_edu_qa_v2_repeat.py --shard_pattern "min_0_to_2000/000*level_2*" --n_qa_pairs=5 --vllm_model=google/gemma-3-12b-it
 ```
+
+2. Self-generated response QA data (depends on step 0 and 1)
+```bash
+# Example commands using gemma-2-2b-it
+# self-gen data for fw_qa_v2
+# 000_0000[0-1] to 000_0000[8-9] for small split
+uv run python data/self_generate_qa.py --vllm_model google/gemma-2-2b-it --glob_pattern 'data/raw_datasets/fw_qa_v2/013*'
+
+# self-gen data for other ds listed in qa_no_fw.yaml
+uv run python data/self_generate_qa.py --vllm_model google/gemma-2-2b-it --config configs/qa_no_fw.yaml
+```
+
+
 <!-- ### Data Generation
 ```bash
 # [WIP] there are other datasets where we used openAI to gen QA pairs

data/self_generate_qa.py

@@ -13,9 +13,10 @@ from ctx_to_lora.data.processing import load_and_process_dataset
 from ctx_to_lora.utils import clear_gpu
 
 SYSTEM_TEMPLATE = (
-    "### System Instructions ###\n"
+    "### SYSTEM INSTRUCTIONS ###\n"
     "You are a creative and helpful assistant.\n"
-    "**DO NOT** hallucinate or make up information."
+    "**DO NOT** hallucinate or make up information.\n"
+    "### END OF SYSTEM INSTRUCTIONS ###"
 )
 
 PROMPT_TEMPLATE = "### Context ###\n{context}\n\n\n### Question ###\n{question}"
@@ -81,7 +82,7 @@ def get_dataset_configs(
 
 
 def create_messages(
-    ctxs: list[str], questions: list[str], vllm_model: str, system_template: str
+    ctxs: list[str], questions: list[list[str]], vllm_model: str, system_template: str
 ) -> list[list[dict]]:
     """Create chat messages for the model."""
     if "gemma" in vllm_model:
@@ -93,7 +94,8 @@ def create_messages(
                     "content": system_template + "\n\n\n" + get_prompt(ctx, q),
                 }
             ]
-            for ctx, q in zip(ctxs, questions)
+            for ctx, q_list in zip(ctxs, questions)
+            for q in q_list
         ]
     else:
         return [
@@ -101,7 +103,8 @@ def create_messages(
                 {"role": "system", "content": system_template},
                 {"role": "user", "content": get_prompt(ctx, q)},
             ]
-            for ctx, q in zip(ctxs, questions)
+            for ctx, q_list in zip(ctxs, questions)
+            for q in q_list
         ]
 
 
@@ -139,18 +142,16 @@ def self_generate(
         ds = ds.take(10)
 
     ctxs = [sample["context"] for sample in ds]
-    questions = [sample["prompt"] for sample in ds]
+    questions = [sample["prompts"] for sample in ds]
 
-    messages = create_messages(ctxs, questions, args.vllm_model, system_template)
+    messages = create_messages(ctxs, questions, args.vllm_model, SYSTEM_TEMPLATE)
 
     print(f"Generating from {len(messages)} contexts")
     completions = llm.chat(
         messages,
         sampling_params=SamplingParams(
             max_tokens=2048,
-            temperature=0.1
-            if args.vllm_model == "mistralai/Mistral-Small-3.1-24B-Instruct-2503"
-            else 1.0,
+            temperature=1.0,
         ),
     )
 
@@ -243,24 +244,6 @@ if __name__ == "__main__":
         max_model_len=MODEL_CTX_LEN.get(vllm_model),
     )
 
-    system_template = SYSTEM_TEMPLATE
-    if vllm_model == "mistralai/Mistral-Small-3.1-24B-Instruct-2503":
-        mm_kwargs = {"image": 0}
-        llm_kwargs.update(
-            {
-                "tokenizer_mode": "mistral",
-                "config_format": "mistral",
-                "load_format": "mistral",
-                "max_model_len": 2**14,
-                "limit_mm_per_prompt": mm_kwargs,
-            }
-        )
-        system_template = (
-            "You are Mistral Small 3.1, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\n"
-            "You power an AI assistant called Le Chat.\n"
-            "Your knowledge base was last updated on 2023-10-01.\n\n\n"
-        ) + SYSTEM_TEMPLATE
-
     print(f"{llm_kwargs=}")
     llm = LLM(**llm_kwargs)
 
@@ -276,7 +259,7 @@ if __name__ == "__main__":
         # Process each dataset
         for ds_name, split in dataset_configs:
             print(f"Processing dataset: {ds_name}, split: {split}")
-            self_generate(ds_name, split, args, llm, system_template)
+            self_generate(ds_name, split, args, llm, SYSTEM_TEMPLATE)
     else:
         assert args.glob_pattern, (
             "glob_pattern must be provided if no ds_names or config"
@@ -290,5 +273,5 @@ if __name__ == "__main__":
                 split=args.split,
                 args=args,
                 llm=llm,
-                system_template=system_template,
+                system_template=SYSTEM_TEMPLATE,
             )