apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

tiny data def

Browse source
This commit is contained in:
51616 2025-07-01 05:52:47 +00:00
parent 375c50757c
commit 92ba1ca76c
4 changed files with 12 additions and 10 deletions
Download patch file Download diff file Expand all files Collapse all files

2
data/generate_fw_edu_qa_v2.py
View file

@ -8,7 +8,7 @@ from datasets import Dataset, load_dataset
from vllm import LLM, SamplingParams
STOP_STRINGS = {
"google/gemma-2-2b-it": ["<eos>", "<end_of_turn>"],
"google/gemma-3-12b-it": ["<eos>", "<end_of_turn>"],
}
SYSTEM_TEMPLATE = (

2
data/generate_fw_edu_qa_v2_repeat.py
View file

@ -8,7 +8,7 @@ from datasets import load_dataset
from vllm import LLM, SamplingParams
STOP_STRINGS = {
"google/gemma-2-2b-it": ["<eos>", "<end_of_turn>"],
"google/gemma-3-12b-it": ["<eos>", "<end_of_turn>"],
}
SYSTEM_TEMPLATE = (

2
data/self_generate_qa.py
View file

@ -138,6 +138,8 @@ def self_generate(
temp = 0.0
if "_temp_" in ds_name:
temp = float(ds_name.split("_temp_")[-1].split("/")[0])
print(f"Processing dataset: {ds_name}, split: {split}")
print(f"Using temperature: {temp}")
kwargs = dict(
ds_name=ds_name,
split=split,

16
src/ctx_to_lora/data/definitions.py
View file

@ -150,9 +150,9 @@ DS_KWARGS = {
train=dict(
path="parquet",
data_files=glob(
"data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_0.parquet"
"data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_0.parquet"
),
split="train[:400000]",
split="train",
),
),
"fw_qa_v2_2k_len_level_0": dict(
@ -187,9 +187,9 @@ DS_KWARGS = {
train=dict(
path="parquet",
data_files=glob(
"data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_1.parquet"
"data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_1.parquet"
),
split="train[:200000]",
split="train",
),
),
"fw_qa_v2_2k_len_level_2": dict(
@ -210,9 +210,9 @@ DS_KWARGS = {
train=dict(
path="parquet",
data_files=glob(
"data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_2.parquet"
"data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_2.parquet"
),
split="train[:100000]",
split="train",
),
),
"fw_qa_v2_2k_len_level_3": dict(
@ -233,9 +233,9 @@ DS_KWARGS = {
train=dict(
path="parquet",
data_files=glob(
"data/raw_datasets/fw_qa_v2/min_0_to_2000/013*level_3.parquet"
"data/raw_datasets/fw_qa_v2/min_0_to_2000/000*level_3.parquet"
),
split="train[:50000]",
split="train",
),
),
# "fw_qa_3_small": dict(