diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml new file mode 100644 index 0000000..c25191f --- /dev/null +++ b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.3.yaml @@ -0,0 +1,52 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-3-1b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.01 + +warmup_steps: 100 + +dataloader_prefetch_factor: 16 +dataloader_num_workers: 8 + +# LoRA +lora_r: 8 +lora_dropout: 0.0 +target_modules: + - down_proj + +# data +train_ds_names: + - self_gen/google/gemma-2-2b-it_temp_0.3/fw_qa_v2_2k_len_level_1_tiny + - self_gen/google/gemma-2-2b-it_temp_0.3/squad_compact + - self_gen/google/gemma-2-2b-it_temp_0.3/pwc_compact + - self_gen/google/gemma-2-2b-it_temp_0.3/ropes_compact + - self_gen/google/gemma-2-2b-it_temp_0.3/drop_compact + +val_ds_names: + - squad + - pwc + - drop + - ropes diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml new file mode 100644 index 0000000..b50322f --- /dev/null +++ b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.5.yaml @@ -0,0 +1,52 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-3-1b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.01 + +warmup_steps: 100 + +dataloader_prefetch_factor: 16 +dataloader_num_workers: 8 + +# LoRA +lora_r: 8 +lora_dropout: 0.0 +target_modules: + - down_proj + +# data +train_ds_names: + - self_gen/google/gemma-2-2b-it_temp_0.5/fw_qa_v2_2k_len_level_1_tiny + - self_gen/google/gemma-2-2b-it_temp_0.5/squad_compact + - self_gen/google/gemma-2-2b-it_temp_0.5/pwc_compact + - self_gen/google/gemma-2-2b-it_temp_0.5/ropes_compact + - self_gen/google/gemma-2-2b-it_temp_0.5/drop_compact + +val_ds_names: + - squad + - pwc + - drop + - ropes diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml new file mode 100644 index 0000000..58fd4bd --- /dev/null +++ b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_0.7.yaml @@ -0,0 +1,52 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-3-1b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.01 + +warmup_steps: 100 + +dataloader_prefetch_factor: 16 +dataloader_num_workers: 8 + +# LoRA +lora_r: 8 +lora_dropout: 0.0 +target_modules: + - down_proj + +# data +train_ds_names: + - self_gen/google/gemma-2-2b-it_temp_0.7/fw_qa_v2_2k_len_level_1_tiny + - self_gen/google/gemma-2-2b-it_temp_0.7/squad_compact + - self_gen/google/gemma-2-2b-it_temp_0.7/pwc_compact + - self_gen/google/gemma-2-2b-it_temp_0.7/ropes_compact + - self_gen/google/gemma-2-2b-it_temp_0.7/drop_compact + +val_ds_names: + - squad + - pwc + - drop + - ropes diff --git a/configs/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml new file mode 100644 index 0000000..787b9b2 --- /dev/null +++ b/configs/qa_short_ctx_self_gen_lv1_tiny_temp_1.0.yaml @@ -0,0 +1,52 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: google/gemma-3-1b-it +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 8 +per_device_eval_batch_size: 8 +max_val_samples_per_ds: 1000 +# optim: schedule_free_adamw + +learning_rate: 0.00004 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.01 + +warmup_steps: 100 + +dataloader_prefetch_factor: 16 +dataloader_num_workers: 8 + +# LoRA +lora_r: 8 +lora_dropout: 0.0 +target_modules: + - down_proj + +# data +train_ds_names: + - self_gen/google/gemma-2-2b-it_temp_1.0/fw_qa_v2_2k_len_level_1_tiny + - self_gen/google/gemma-2-2b-it_temp_1.0/squad_compact + - self_gen/google/gemma-2-2b-it_temp_1.0/pwc_compact + - self_gen/google/gemma-2-2b-it_temp_1.0/ropes_compact + - self_gen/google/gemma-2-2b-it_temp_1.0/drop_compact + +val_ds_names: + - squad + - pwc + - drop + - ropes diff --git a/data/self_generate_qa.py b/data/self_generate_qa.py index 6e6e7ab..0a7ed33 100644 --- a/data/self_generate_qa.py +++ b/data/self_generate_qa.py @@ -74,8 +74,13 @@ def get_dataset_configs( # Process train datasets train_ds_names = config.get("train_ds_names", []) + # self_gen_train_ds_names = [ + # (ds_name.split("/")[-1], "train") + # for ds_name in train_ds_names + # if ds_name.startswith("self_gen/") + # ] self_gen_train_ds_names = [ - (ds_name.split("/")[-1], "train") + (ds_name, "train") for ds_name in train_ds_names if ds_name.startswith("self_gen/") ] @@ -140,14 +145,7 @@ def self_generate( temp = float(ds_name.split("_temp_")[-1].split("/")[0]) print(f"Processing dataset: {ds_name}, split: {split}") print(f"Using temperature: {temp}") - kwargs = dict( - ds_name=ds_name, - split=split, - add_negative_prompt=False, - add_repeat_prompt=False, - repeat_prob=0, - is_pretrain=False, - ) + if parquet_file: print(f"Loading dataset from parquet file: {parquet_file}") @@ -163,6 +161,14 @@ def self_generate( ds_name = ds_name.split("/")[-1] # Extract just the dataset name print(f"Loading dataset: {ds_name} with split: {split}") + kwargs = dict( + ds_name=ds_name, + split=split, + add_negative_prompt=False, + add_repeat_prompt=False, + repeat_prob=0, + is_pretrain=False, + ) ds = load_and_process_dataset(**kwargs, streaming=False, num_proc=8)