From c7f7a2f788adc2d72ea74e60807d50615ae58e84 Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 7 Jan 2025 12:25:47 +0000 Subject: [PATCH] new configs --- configs/context_numbers_128.yaml | 2 +- configs/context_numbers_256.yaml | 2 +- configs/context_numbers_256_new.yaml | 68 ++++++++++++++ configs/context_numbers_256_only.yaml | 52 +++++++++++ configs/pwc_and_ctx_numbers_256.yaml | 130 ++------------------------ 5 files changed, 131 insertions(+), 123 deletions(-) create mode 100644 configs/context_numbers_256_new.yaml create mode 100644 configs/context_numbers_256_only.yaml diff --git a/configs/context_numbers_128.yaml b/configs/context_numbers_128.yaml index 72b262c..3c5c5f2 100644 --- a/configs/context_numbers_128.yaml +++ b/configs/context_numbers_128.yaml @@ -17,7 +17,7 @@ label_names: ["labels"] # batch_eval_metrics: true per_device_train_batch_size: 64 -per_device_eval_batch_size: 64 +per_device_eval_batch_size: 1 max_val_samples_per_ds: 50 # optim: schedule_free_adamw learning_rate: 0.00001 diff --git a/configs/context_numbers_256.yaml b/configs/context_numbers_256.yaml index 04de027..92cba34 100644 --- a/configs/context_numbers_256.yaml +++ b/configs/context_numbers_256.yaml @@ -17,7 +17,7 @@ label_names: ["labels"] # batch_eval_metrics: true per_device_train_batch_size: 32 -per_device_eval_batch_size: 32 +per_device_eval_batch_size: 1 max_val_samples_per_ds: 20 # optim: schedule_free_adamw learning_rate: 0.00001 diff --git a/configs/context_numbers_256_new.yaml b/configs/context_numbers_256_new.yaml new file mode 100644 index 0000000..03e1e2f --- /dev/null +++ b/configs/context_numbers_256_new.yaml @@ -0,0 +1,68 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: meta-llama/Llama-3.2-1B-Instruct +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 32 +per_device_eval_batch_size: 1 +max_val_samples_per_ds: 20 +# optim: schedule_free_adamw +learning_rate: 0.00001 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.1 +warmup_ratio: 0.05 + +# LoRA +lora_r: 16 +lora_dropout: 0.05 +target_modules: + - down_proj + - up_proj + - gate_proj + +# data +train_ds_names: +- data/raw_datasets/context_numbers_4 +- data/raw_datasets/context_numbers_8 +- data/raw_datasets/context_numbers_16 +- data/raw_datasets/context_numbers_32 +- data/raw_datasets/context_numbers_64 +- data/raw_datasets/context_numbers_80 +- data/raw_datasets/context_numbers_96 +- data/raw_datasets/context_numbers_112 +- data/raw_datasets/context_numbers_128 +- data/raw_datasets/context_numbers_144 +- data/raw_datasets/context_numbers_160 +- data/raw_datasets/context_numbers_176 +- data/raw_datasets/context_numbers_192 +- data/raw_datasets/context_numbers_208 +- data/raw_datasets/context_numbers_224 +- data/raw_datasets/context_numbers_240 +- data/raw_datasets/context_numbers_256 + +val_ds_names: +- data/raw_datasets/context_numbers_16 +- data/raw_datasets/context_numbers_32 +- data/raw_datasets/context_numbers_64 +- data/raw_datasets/context_numbers_128 +- data/raw_datasets/context_numbers_256 + +test_ds_names: +- data/raw_datasets/context_numbers_512 +- data/raw_datasets/context_numbers_1024 +- data/raw_datasets/context_numbers_2048 + diff --git a/configs/context_numbers_256_only.yaml b/configs/context_numbers_256_only.yaml new file mode 100644 index 0000000..1ebff0c --- /dev/null +++ b/configs/context_numbers_256_only.yaml @@ -0,0 +1,52 @@ +output_dir: "" # just a placeholder +bf16: true +model_name_or_path: meta-llama/Llama-3.2-1B-Instruct +label_names: ["labels"] +# eval_on_start: True +# eval_strategy: "steps" +# eval_steps: 500 +# save_strategy: "no" +# # save_steps: 500 +# logging_strategy: "steps" +# logging_steps: 100 +# use_liger_kernel: true +# remove_unused_columns: false + +# needed to avoid OOM by compute the metrics batch by batch +# w/o this the trainer stores logits of all sample in memory... +# batch_eval_metrics: true + +per_device_train_batch_size: 32 +per_device_eval_batch_size: 1 +max_val_samples_per_ds: 20 +# optim: schedule_free_adamw +learning_rate: 0.00001 +# lr_scheduler_type: "constant_with_warmup" +neftune_noise_alpha: 1 +weight_decay: 0.1 +warmup_ratio: 0.05 + +# LoRA +lora_r: 16 +lora_dropout: 0.05 +target_modules: + - down_proj + - up_proj + - gate_proj + +# data +train_ds_names: +- data/raw_datasets/context_numbers_256 + +val_ds_names: +- data/raw_datasets/context_numbers_16 +- data/raw_datasets/context_numbers_32 +- data/raw_datasets/context_numbers_64 +- data/raw_datasets/context_numbers_128 +- data/raw_datasets/context_numbers_256 + +test_ds_names: +- data/raw_datasets/context_numbers_512 +- data/raw_datasets/context_numbers_1024 +- data/raw_datasets/context_numbers_2048 + diff --git a/configs/pwc_and_ctx_numbers_256.yaml b/configs/pwc_and_ctx_numbers_256.yaml index e3fe688..fd96c31 100644 --- a/configs/pwc_and_ctx_numbers_256.yaml +++ b/configs/pwc_and_ctx_numbers_256.yaml @@ -17,7 +17,7 @@ label_names: ["labels"] # batch_eval_metrics: true per_device_train_batch_size: 32 -per_device_eval_batch_size: 32 +per_device_eval_batch_size: 1 max_val_samples_per_ds: 20 # optim: schedule_free_adamw learning_rate: 0.00001 @@ -37,134 +37,22 @@ target_modules: # data train_ds_names: - sggetao/PwC +- data/raw_datasets/context_numbers_4 +- data/raw_datasets/context_numbers_8 +- data/raw_datasets/context_numbers_16 +- data/raw_datasets/context_numbers_32 +- data/raw_datasets/context_numbers_64 +- data/raw_datasets/context_numbers_80 +- data/raw_datasets/context_numbers_96 +- data/raw_datasets/context_numbers_112 - data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_129 -- data/raw_datasets/context_numbers_130 -- data/raw_datasets/context_numbers_131 -- data/raw_datasets/context_numbers_132 -- data/raw_datasets/context_numbers_133 -- data/raw_datasets/context_numbers_134 -- data/raw_datasets/context_numbers_135 -- data/raw_datasets/context_numbers_136 -- data/raw_datasets/context_numbers_137 -- data/raw_datasets/context_numbers_138 -- data/raw_datasets/context_numbers_139 -- data/raw_datasets/context_numbers_140 -- data/raw_datasets/context_numbers_141 -- data/raw_datasets/context_numbers_142 -- data/raw_datasets/context_numbers_143 - data/raw_datasets/context_numbers_144 -- data/raw_datasets/context_numbers_145 -- data/raw_datasets/context_numbers_146 -- data/raw_datasets/context_numbers_147 -- data/raw_datasets/context_numbers_148 -- data/raw_datasets/context_numbers_149 -- data/raw_datasets/context_numbers_150 -- data/raw_datasets/context_numbers_151 -- data/raw_datasets/context_numbers_152 -- data/raw_datasets/context_numbers_153 -- data/raw_datasets/context_numbers_154 -- data/raw_datasets/context_numbers_155 -- data/raw_datasets/context_numbers_156 -- data/raw_datasets/context_numbers_157 -- data/raw_datasets/context_numbers_158 -- data/raw_datasets/context_numbers_159 - data/raw_datasets/context_numbers_160 -- data/raw_datasets/context_numbers_161 -- data/raw_datasets/context_numbers_162 -- data/raw_datasets/context_numbers_163 -- data/raw_datasets/context_numbers_164 -- data/raw_datasets/context_numbers_165 -- data/raw_datasets/context_numbers_166 -- data/raw_datasets/context_numbers_167 -- data/raw_datasets/context_numbers_168 -- data/raw_datasets/context_numbers_169 -- data/raw_datasets/context_numbers_170 -- data/raw_datasets/context_numbers_171 -- data/raw_datasets/context_numbers_172 -- data/raw_datasets/context_numbers_173 -- data/raw_datasets/context_numbers_174 -- data/raw_datasets/context_numbers_175 - data/raw_datasets/context_numbers_176 -- data/raw_datasets/context_numbers_177 -- data/raw_datasets/context_numbers_178 -- data/raw_datasets/context_numbers_179 -- data/raw_datasets/context_numbers_180 -- data/raw_datasets/context_numbers_181 -- data/raw_datasets/context_numbers_182 -- data/raw_datasets/context_numbers_183 -- data/raw_datasets/context_numbers_184 -- data/raw_datasets/context_numbers_185 -- data/raw_datasets/context_numbers_186 -- data/raw_datasets/context_numbers_187 -- data/raw_datasets/context_numbers_188 -- data/raw_datasets/context_numbers_189 -- data/raw_datasets/context_numbers_190 -- data/raw_datasets/context_numbers_191 - data/raw_datasets/context_numbers_192 -- data/raw_datasets/context_numbers_193 -- data/raw_datasets/context_numbers_194 -- data/raw_datasets/context_numbers_195 -- data/raw_datasets/context_numbers_196 -- data/raw_datasets/context_numbers_197 -- data/raw_datasets/context_numbers_198 -- data/raw_datasets/context_numbers_199 -- data/raw_datasets/context_numbers_200 -- data/raw_datasets/context_numbers_201 -- data/raw_datasets/context_numbers_202 -- data/raw_datasets/context_numbers_203 -- data/raw_datasets/context_numbers_204 -- data/raw_datasets/context_numbers_205 -- data/raw_datasets/context_numbers_206 -- data/raw_datasets/context_numbers_207 - data/raw_datasets/context_numbers_208 -- data/raw_datasets/context_numbers_209 -- data/raw_datasets/context_numbers_210 -- data/raw_datasets/context_numbers_211 -- data/raw_datasets/context_numbers_212 -- data/raw_datasets/context_numbers_213 -- data/raw_datasets/context_numbers_214 -- data/raw_datasets/context_numbers_215 -- data/raw_datasets/context_numbers_216 -- data/raw_datasets/context_numbers_217 -- data/raw_datasets/context_numbers_218 -- data/raw_datasets/context_numbers_219 -- data/raw_datasets/context_numbers_220 -- data/raw_datasets/context_numbers_221 -- data/raw_datasets/context_numbers_222 -- data/raw_datasets/context_numbers_223 - data/raw_datasets/context_numbers_224 -- data/raw_datasets/context_numbers_225 -- data/raw_datasets/context_numbers_226 -- data/raw_datasets/context_numbers_227 -- data/raw_datasets/context_numbers_228 -- data/raw_datasets/context_numbers_229 -- data/raw_datasets/context_numbers_230 -- data/raw_datasets/context_numbers_231 -- data/raw_datasets/context_numbers_232 -- data/raw_datasets/context_numbers_233 -- data/raw_datasets/context_numbers_234 -- data/raw_datasets/context_numbers_235 -- data/raw_datasets/context_numbers_236 -- data/raw_datasets/context_numbers_237 -- data/raw_datasets/context_numbers_238 -- data/raw_datasets/context_numbers_239 - data/raw_datasets/context_numbers_240 -- data/raw_datasets/context_numbers_241 -- data/raw_datasets/context_numbers_242 -- data/raw_datasets/context_numbers_243 -- data/raw_datasets/context_numbers_244 -- data/raw_datasets/context_numbers_245 -- data/raw_datasets/context_numbers_246 -- data/raw_datasets/context_numbers_247 -- data/raw_datasets/context_numbers_248 -- data/raw_datasets/context_numbers_249 -- data/raw_datasets/context_numbers_250 -- data/raw_datasets/context_numbers_251 -- data/raw_datasets/context_numbers_252 -- data/raw_datasets/context_numbers_253 -- data/raw_datasets/context_numbers_254 -- data/raw_datasets/context_numbers_255 - data/raw_datasets/context_numbers_256