From c983a3a0d8a9fc6c3b4890f108b3d8d920281856 Mon Sep 17 00:00:00 2001
From: 51616 <rujikorn.ch@gmail.com>
Date: Mon, 20 Jan 2025 09:18:07 +0000
Subject: [PATCH] more data

---
 configs/pretrain_all.yaml     | 62 +++++++++++++++++++++++++++++++++++
 src/ctx_to_lora/data_utils.py |  8 ++---
 2 files changed, 64 insertions(+), 6 deletions(-)
 create mode 100644 configs/pretrain_all.yaml

diff --git a/configs/pretrain_all.yaml b/configs/pretrain_all.yaml
new file mode 100644
index 0000000..86a22b6
--- /dev/null
+++ b/configs/pretrain_all.yaml
@@ -0,0 +1,62 @@
+output_dir: "" # just a placeholder
+bf16: true
+model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
+label_names: ["labels"]
+# eval_on_start: True
+# eval_strategy: "steps"
+# eval_steps: 500
+# save_strategy: "no"
+# # save_steps: 500
+# logging_strategy: "steps"
+# logging_steps: 100
+# use_liger_kernel: true
+# remove_unused_columns: false
+
+# needed to avoid OOM by compute the metrics batch by batch
+# w/o this the trainer stores logits of all sample in memory...
+# batch_eval_metrics: true
+
+per_device_train_batch_size: 8
+per_device_eval_batch_size: 8
+max_val_samples_per_ds: 1000
+# optim: schedule_free_adamw
+
+learning_rate: 0.00002
+# lr_scheduler_type: "constant_with_warmup"
+neftune_noise_alpha: 5
+weight_decay: 0.01
+# warmup_ratio: 0.1
+warmup_steps: 100
+
+dataloader_prefetch_factor: 8
+dataloader_num_workers: 8
+# LoRA
+lora_r: 8
+lora_dropout: 0.05
+target_modules:
+  - down_proj
+  - up_proj
+
+# data
+train_ds_names:
+- fw_qa
+- fw_qa_large
+- ctx_qa
+- pwc
+- hotpot_qa
+- squad
+- drop
+- narrativeqa
+- quoref
+- ropes
+- synthetic_convqa
+
+val_ds_names:
+- fw_qa_large
+- ctx_qa
+- pwc
+- hotpot_qa
+- squad
+
+load_best_model_at_end: true
+metric_for_best_model: eval_pwc_loss
diff --git a/src/ctx_to_lora/data_utils.py b/src/ctx_to_lora/data_utils.py
index 00b2e42..fb28af3 100644
--- a/src/ctx_to_lora/data_utils.py
+++ b/src/ctx_to_lora/data_utils.py
@@ -88,11 +88,7 @@ DS_KWARGS = {
     "drop": dict(
         train=dict(
             path="ucinlp/drop",
-            split="train[900:]",
-        ),
-        validation=dict(
-            path="ucinlp/drop",
-            split="train[:900]",
+            split="train",
         ),
     ),
     "narrativeqa": dict(
@@ -138,7 +134,7 @@ DS_KWARGS = {
     "openmathintx-2": dict(
         train=dict(
             path="nvidia/OpenMathInstruct-2",
-            split="train_1M[:100000]",
+            split="train_1M[:500000]",
         ),
     ),
 }