From 9411b81b7f3bc27f1dc9e6f48d8de13f557ede30 Mon Sep 17 00:00:00 2001 From: 51616 Date: Thu, 10 Jul 2025 10:04:45 +0000 Subject: [PATCH] self-gen parquet format --- .../qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml | 2 +- .../qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml | 2 +- .../qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml | 2 +- src/ctx_to_lora/data/processing.py | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/configs/small_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml b/configs/small_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml index 8f9f671..60c8a68 100644 --- a/configs/small_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml +++ b/configs/small_exp/qa_short_ctx_self_gen_lv1_closed_qa_1_small.yaml @@ -39,7 +39,7 @@ target_modules: # data train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/*level_0.parquet + - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/train/*level_0.parquet - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact # these provide exact tokens needed, no need to use self-gen data - squad_compact diff --git a/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml b/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml index 6c965fb..17377e7 100644 --- a/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml +++ b/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_0.5_small.yaml @@ -39,7 +39,7 @@ target_modules: # data train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.5/fw_qa_v2/min_0_to_2000/*level_3.parquet + - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.5/fw_qa_v2/min_0_to_2000/train/*level_3.parquet - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact # these provide exact tokens needed, no need to use self-gen data - squad_compact diff --git a/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml b/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml index 0c5e25f..15dd59f 100644 --- a/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml +++ b/configs/small_exp/qa_short_ctx_self_gen_lv3_closed_qa_1_small.yaml @@ -39,7 +39,7 @@ target_modules: # data train_ds_names: - - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/*level_3.parquet + - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_1.0/fw_qa_v2/min_0_to_2000/train/*level_3.parquet - self_gen/google/gemma-2-2b-it_temp_0.0_closed_qa_prob_0.0/pwc_compact # these provide exact tokens needed, no need to use self-gen data - squad_compact diff --git a/src/ctx_to_lora/data/processing.py b/src/ctx_to_lora/data/processing.py index 5c3d85e..84851fe 100644 --- a/src/ctx_to_lora/data/processing.py +++ b/src/ctx_to_lora/data/processing.py @@ -17,6 +17,7 @@ from ctx_to_lora.data.definitions import ( DS_KWARGS, EVAL_INTX_TEMPLATES, IGNORE_INDEX, + RAW_DATA_DIR, REPEAT_PROMPTS, SELF_GEN_DATA_DIR, TRANSFORMED_DATA_DIR, @@ -323,10 +324,10 @@ def get_ds_kwargs(ds_name: str, split: str) -> dict[str, Any]: if ds_name.startswith("self_gen/"): if ds_name.endswith(".parquet"): # ds_name is a glob pattern - files = glob(ds_name) + files = glob(f"{RAW_DATA_DIR}/{ds_name}") if not files: raise FileNotFoundError( - f"The provided pattern does not match any files: {ds_name}" + f"The provided pattern does not match any files: {RAW_DATA_DIR}/{ds_name}" ) else: # e.g., "self_gen/google/gemma-2-2b-it/pwc"