apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

self-gen parquet format

Browse source
This commit is contained in:
51616 2025-07-10 10:04:45 +00:00
parent f60d98aa8b
commit 9411b81b7f
4 changed files with 6 additions and 5 deletions
Download patch file Download diff file Expand all files Collapse all files

2
configs/small_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml
View file

@ -39,7 +39,7 @@ target_modules:
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/*level_0.parquet
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/train/*level_0.parquet
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
# these provide exact tokens needed, no need to use self-gen data
- squad_compact

2
configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml
View file

@ -39,7 +39,7 @@ target_modules:
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.5/fw_qa_v2/min_0_to_2000/*level_3.parquet
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.5/fw_qa_v2/min_0_to_2000/train/*level_3.parquet
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
# these provide exact tokens needed, no need to use self-gen data
- squad_compact

2
configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml
View file

@ -39,7 +39,7 @@ target_modules:
# data
train_ds_names:
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/*level_3.parquet
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/train/*level_3.parquet
- self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact
# these provide exact tokens needed, no need to use self-gen data
- squad_compact

5
src/ctx_to_lora/data/processing.py
View file

@ -17,6 +17,7 @@ from ctx_to_lora.data.definitions import (
DS_KWARGS,
EVAL_INTX_TEMPLATES,
IGNORE_INDEX,
RAW_DATA_DIR,
REPEAT_PROMPTS,
SELF_GEN_DATA_DIR,
TRANSFORMED_DATA_DIR,
@ -323,10 +324,10 @@ def get_ds_kwargs(ds_name: str, split: str) -> dict[str, Any]:
if ds_name.startswith("self_gen/"):
if ds_name.endswith(".parquet"):
# ds_name is a glob pattern
files = glob(ds_name)
files = glob(f"{RAW_DATA_DIR}/{ds_name}")
if not files:
raise FileNotFoundError(
f"The provided pattern does not match any files: {ds_name}"
f"The provided pattern does not match any files: {RAW_DATA_DIR}/{ds_name}"
)
else:
# e.g., "self_gen/google/gemma-2-2b-it/pwc"