mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
limit max test sample + config test up to 256 number
This commit is contained in:
51616
parent
2a4e5742ca
commit
6c579fa373
6 changed files with 25 additions and 383 deletions
|
|
@ -1,62 +0,0 @@
|
|||
output_dir: "" # just a placeholder
|
||||
bf16: true
|
||||
model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
|
||||
label_names: ["labels"]
|
||||
# eval_on_start: True
|
||||
# eval_strategy: "steps"
|
||||
# eval_steps: 500
|
||||
# save_strategy: "no"
|
||||
# # save_steps: 500
|
||||
# logging_strategy: "steps"
|
||||
# logging_steps: 100
|
||||
# use_liger_kernel: true
|
||||
# remove_unused_columns: false
|
||||
|
||||
# needed to avoid OOM by compute the metrics batch by batch
|
||||
# w/o this the trainer stores logits of all sample in memory...
|
||||
# batch_eval_metrics: true
|
||||
|
||||
per_device_train_batch_size: 32
|
||||
per_device_eval_batch_size: 1
|
||||
max_val_samples_per_ds: 20
|
||||
# optim: schedule_free_adamw
|
||||
learning_rate: 0.00001
|
||||
# lr_scheduler_type: "constant_with_warmup"
|
||||
neftune_noise_alpha: 1
|
||||
weight_decay: 0.1
|
||||
warmup_ratio: 0.05
|
||||
|
||||
# LoRA
|
||||
lora_r: 16
|
||||
lora_dropout: 0.05
|
||||
target_modules:
|
||||
- down_proj
|
||||
- up_proj
|
||||
- gate_proj
|
||||
|
||||
# data
|
||||
train_ds_names:
|
||||
- data/raw_datasets/context_numbers_2
|
||||
- data/raw_datasets/context_numbers_4
|
||||
- data/raw_datasets/context_numbers_8
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_48
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_80
|
||||
- data/raw_datasets/context_numbers_96
|
||||
- data/raw_datasets/context_numbers_112
|
||||
- data/raw_datasets/context_numbers_128
|
||||
|
||||
val_ds_names:
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_128
|
||||
- data/raw_datasets/context_numbers_256
|
||||
|
||||
test_ds_names:
|
||||
- data/raw_datasets/context_numbers_512
|
||||
- data/raw_datasets/context_numbers_1024
|
||||
- data/raw_datasets/context_numbers_2048
|
||||
|
||||
|
|
@ -36,260 +36,16 @@ target_modules:
|
|||
|
||||
# data
|
||||
train_ds_names:
|
||||
- data/raw_datasets/context_numbers_2
|
||||
- data/raw_datasets/context_numbers_3
|
||||
- data/raw_datasets/context_numbers_4
|
||||
- data/raw_datasets/context_numbers_5
|
||||
- data/raw_datasets/context_numbers_6
|
||||
- data/raw_datasets/context_numbers_7
|
||||
- data/raw_datasets/context_numbers_8
|
||||
- data/raw_datasets/context_numbers_9
|
||||
- data/raw_datasets/context_numbers_10
|
||||
- data/raw_datasets/context_numbers_11
|
||||
- data/raw_datasets/context_numbers_12
|
||||
- data/raw_datasets/context_numbers_13
|
||||
- data/raw_datasets/context_numbers_14
|
||||
- data/raw_datasets/context_numbers_15
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_17
|
||||
- data/raw_datasets/context_numbers_18
|
||||
- data/raw_datasets/context_numbers_19
|
||||
- data/raw_datasets/context_numbers_20
|
||||
- data/raw_datasets/context_numbers_21
|
||||
- data/raw_datasets/context_numbers_22
|
||||
- data/raw_datasets/context_numbers_23
|
||||
- data/raw_datasets/context_numbers_24
|
||||
- data/raw_datasets/context_numbers_25
|
||||
- data/raw_datasets/context_numbers_26
|
||||
- data/raw_datasets/context_numbers_27
|
||||
- data/raw_datasets/context_numbers_28
|
||||
- data/raw_datasets/context_numbers_29
|
||||
- data/raw_datasets/context_numbers_30
|
||||
- data/raw_datasets/context_numbers_31
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_33
|
||||
- data/raw_datasets/context_numbers_34
|
||||
- data/raw_datasets/context_numbers_35
|
||||
- data/raw_datasets/context_numbers_36
|
||||
- data/raw_datasets/context_numbers_37
|
||||
- data/raw_datasets/context_numbers_38
|
||||
- data/raw_datasets/context_numbers_39
|
||||
- data/raw_datasets/context_numbers_40
|
||||
- data/raw_datasets/context_numbers_41
|
||||
- data/raw_datasets/context_numbers_42
|
||||
- data/raw_datasets/context_numbers_43
|
||||
- data/raw_datasets/context_numbers_44
|
||||
- data/raw_datasets/context_numbers_45
|
||||
- data/raw_datasets/context_numbers_46
|
||||
- data/raw_datasets/context_numbers_47
|
||||
- data/raw_datasets/context_numbers_48
|
||||
- data/raw_datasets/context_numbers_49
|
||||
- data/raw_datasets/context_numbers_50
|
||||
- data/raw_datasets/context_numbers_51
|
||||
- data/raw_datasets/context_numbers_52
|
||||
- data/raw_datasets/context_numbers_53
|
||||
- data/raw_datasets/context_numbers_54
|
||||
- data/raw_datasets/context_numbers_55
|
||||
- data/raw_datasets/context_numbers_56
|
||||
- data/raw_datasets/context_numbers_57
|
||||
- data/raw_datasets/context_numbers_58
|
||||
- data/raw_datasets/context_numbers_59
|
||||
- data/raw_datasets/context_numbers_60
|
||||
- data/raw_datasets/context_numbers_61
|
||||
- data/raw_datasets/context_numbers_62
|
||||
- data/raw_datasets/context_numbers_63
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_65
|
||||
- data/raw_datasets/context_numbers_66
|
||||
- data/raw_datasets/context_numbers_67
|
||||
- data/raw_datasets/context_numbers_68
|
||||
- data/raw_datasets/context_numbers_69
|
||||
- data/raw_datasets/context_numbers_70
|
||||
- data/raw_datasets/context_numbers_71
|
||||
- data/raw_datasets/context_numbers_72
|
||||
- data/raw_datasets/context_numbers_73
|
||||
- data/raw_datasets/context_numbers_74
|
||||
- data/raw_datasets/context_numbers_75
|
||||
- data/raw_datasets/context_numbers_76
|
||||
- data/raw_datasets/context_numbers_77
|
||||
- data/raw_datasets/context_numbers_78
|
||||
- data/raw_datasets/context_numbers_79
|
||||
- data/raw_datasets/context_numbers_80
|
||||
- data/raw_datasets/context_numbers_81
|
||||
- data/raw_datasets/context_numbers_82
|
||||
- data/raw_datasets/context_numbers_83
|
||||
- data/raw_datasets/context_numbers_84
|
||||
- data/raw_datasets/context_numbers_85
|
||||
- data/raw_datasets/context_numbers_86
|
||||
- data/raw_datasets/context_numbers_87
|
||||
- data/raw_datasets/context_numbers_88
|
||||
- data/raw_datasets/context_numbers_89
|
||||
- data/raw_datasets/context_numbers_90
|
||||
- data/raw_datasets/context_numbers_91
|
||||
- data/raw_datasets/context_numbers_92
|
||||
- data/raw_datasets/context_numbers_93
|
||||
- data/raw_datasets/context_numbers_94
|
||||
- data/raw_datasets/context_numbers_95
|
||||
- data/raw_datasets/context_numbers_96
|
||||
- data/raw_datasets/context_numbers_97
|
||||
- data/raw_datasets/context_numbers_98
|
||||
- data/raw_datasets/context_numbers_99
|
||||
- data/raw_datasets/context_numbers_100
|
||||
- data/raw_datasets/context_numbers_101
|
||||
- data/raw_datasets/context_numbers_102
|
||||
- data/raw_datasets/context_numbers_103
|
||||
- data/raw_datasets/context_numbers_104
|
||||
- data/raw_datasets/context_numbers_105
|
||||
- data/raw_datasets/context_numbers_106
|
||||
- data/raw_datasets/context_numbers_107
|
||||
- data/raw_datasets/context_numbers_108
|
||||
- data/raw_datasets/context_numbers_109
|
||||
- data/raw_datasets/context_numbers_110
|
||||
- data/raw_datasets/context_numbers_111
|
||||
- data/raw_datasets/context_numbers_112
|
||||
- data/raw_datasets/context_numbers_113
|
||||
- data/raw_datasets/context_numbers_114
|
||||
- data/raw_datasets/context_numbers_115
|
||||
- data/raw_datasets/context_numbers_116
|
||||
- data/raw_datasets/context_numbers_117
|
||||
- data/raw_datasets/context_numbers_118
|
||||
- data/raw_datasets/context_numbers_119
|
||||
- data/raw_datasets/context_numbers_120
|
||||
- data/raw_datasets/context_numbers_121
|
||||
- data/raw_datasets/context_numbers_122
|
||||
- data/raw_datasets/context_numbers_123
|
||||
- data/raw_datasets/context_numbers_124
|
||||
- data/raw_datasets/context_numbers_125
|
||||
- data/raw_datasets/context_numbers_126
|
||||
- data/raw_datasets/context_numbers_127
|
||||
- data/raw_datasets/context_numbers_128
|
||||
- data/raw_datasets/context_numbers_129
|
||||
- data/raw_datasets/context_numbers_130
|
||||
- data/raw_datasets/context_numbers_131
|
||||
- data/raw_datasets/context_numbers_132
|
||||
- data/raw_datasets/context_numbers_133
|
||||
- data/raw_datasets/context_numbers_134
|
||||
- data/raw_datasets/context_numbers_135
|
||||
- data/raw_datasets/context_numbers_136
|
||||
- data/raw_datasets/context_numbers_137
|
||||
- data/raw_datasets/context_numbers_138
|
||||
- data/raw_datasets/context_numbers_139
|
||||
- data/raw_datasets/context_numbers_140
|
||||
- data/raw_datasets/context_numbers_141
|
||||
- data/raw_datasets/context_numbers_142
|
||||
- data/raw_datasets/context_numbers_143
|
||||
- data/raw_datasets/context_numbers_144
|
||||
- data/raw_datasets/context_numbers_145
|
||||
- data/raw_datasets/context_numbers_146
|
||||
- data/raw_datasets/context_numbers_147
|
||||
- data/raw_datasets/context_numbers_148
|
||||
- data/raw_datasets/context_numbers_149
|
||||
- data/raw_datasets/context_numbers_150
|
||||
- data/raw_datasets/context_numbers_151
|
||||
- data/raw_datasets/context_numbers_152
|
||||
- data/raw_datasets/context_numbers_153
|
||||
- data/raw_datasets/context_numbers_154
|
||||
- data/raw_datasets/context_numbers_155
|
||||
- data/raw_datasets/context_numbers_156
|
||||
- data/raw_datasets/context_numbers_157
|
||||
- data/raw_datasets/context_numbers_158
|
||||
- data/raw_datasets/context_numbers_159
|
||||
- data/raw_datasets/context_numbers_160
|
||||
- data/raw_datasets/context_numbers_161
|
||||
- data/raw_datasets/context_numbers_162
|
||||
- data/raw_datasets/context_numbers_163
|
||||
- data/raw_datasets/context_numbers_164
|
||||
- data/raw_datasets/context_numbers_165
|
||||
- data/raw_datasets/context_numbers_166
|
||||
- data/raw_datasets/context_numbers_167
|
||||
- data/raw_datasets/context_numbers_168
|
||||
- data/raw_datasets/context_numbers_169
|
||||
- data/raw_datasets/context_numbers_170
|
||||
- data/raw_datasets/context_numbers_171
|
||||
- data/raw_datasets/context_numbers_172
|
||||
- data/raw_datasets/context_numbers_173
|
||||
- data/raw_datasets/context_numbers_174
|
||||
- data/raw_datasets/context_numbers_175
|
||||
- data/raw_datasets/context_numbers_176
|
||||
- data/raw_datasets/context_numbers_177
|
||||
- data/raw_datasets/context_numbers_178
|
||||
- data/raw_datasets/context_numbers_179
|
||||
- data/raw_datasets/context_numbers_180
|
||||
- data/raw_datasets/context_numbers_181
|
||||
- data/raw_datasets/context_numbers_182
|
||||
- data/raw_datasets/context_numbers_183
|
||||
- data/raw_datasets/context_numbers_184
|
||||
- data/raw_datasets/context_numbers_185
|
||||
- data/raw_datasets/context_numbers_186
|
||||
- data/raw_datasets/context_numbers_187
|
||||
- data/raw_datasets/context_numbers_188
|
||||
- data/raw_datasets/context_numbers_189
|
||||
- data/raw_datasets/context_numbers_190
|
||||
- data/raw_datasets/context_numbers_191
|
||||
- data/raw_datasets/context_numbers_192
|
||||
- data/raw_datasets/context_numbers_193
|
||||
- data/raw_datasets/context_numbers_194
|
||||
- data/raw_datasets/context_numbers_195
|
||||
- data/raw_datasets/context_numbers_196
|
||||
- data/raw_datasets/context_numbers_197
|
||||
- data/raw_datasets/context_numbers_198
|
||||
- data/raw_datasets/context_numbers_199
|
||||
- data/raw_datasets/context_numbers_200
|
||||
- data/raw_datasets/context_numbers_201
|
||||
- data/raw_datasets/context_numbers_202
|
||||
- data/raw_datasets/context_numbers_203
|
||||
- data/raw_datasets/context_numbers_204
|
||||
- data/raw_datasets/context_numbers_205
|
||||
- data/raw_datasets/context_numbers_206
|
||||
- data/raw_datasets/context_numbers_207
|
||||
- data/raw_datasets/context_numbers_208
|
||||
- data/raw_datasets/context_numbers_209
|
||||
- data/raw_datasets/context_numbers_210
|
||||
- data/raw_datasets/context_numbers_211
|
||||
- data/raw_datasets/context_numbers_212
|
||||
- data/raw_datasets/context_numbers_213
|
||||
- data/raw_datasets/context_numbers_214
|
||||
- data/raw_datasets/context_numbers_215
|
||||
- data/raw_datasets/context_numbers_216
|
||||
- data/raw_datasets/context_numbers_217
|
||||
- data/raw_datasets/context_numbers_218
|
||||
- data/raw_datasets/context_numbers_219
|
||||
- data/raw_datasets/context_numbers_220
|
||||
- data/raw_datasets/context_numbers_221
|
||||
- data/raw_datasets/context_numbers_222
|
||||
- data/raw_datasets/context_numbers_223
|
||||
- data/raw_datasets/context_numbers_224
|
||||
- data/raw_datasets/context_numbers_225
|
||||
- data/raw_datasets/context_numbers_226
|
||||
- data/raw_datasets/context_numbers_227
|
||||
- data/raw_datasets/context_numbers_228
|
||||
- data/raw_datasets/context_numbers_229
|
||||
- data/raw_datasets/context_numbers_230
|
||||
- data/raw_datasets/context_numbers_231
|
||||
- data/raw_datasets/context_numbers_232
|
||||
- data/raw_datasets/context_numbers_233
|
||||
- data/raw_datasets/context_numbers_234
|
||||
- data/raw_datasets/context_numbers_235
|
||||
- data/raw_datasets/context_numbers_236
|
||||
- data/raw_datasets/context_numbers_237
|
||||
- data/raw_datasets/context_numbers_238
|
||||
- data/raw_datasets/context_numbers_239
|
||||
- data/raw_datasets/context_numbers_240
|
||||
- data/raw_datasets/context_numbers_241
|
||||
- data/raw_datasets/context_numbers_242
|
||||
- data/raw_datasets/context_numbers_243
|
||||
- data/raw_datasets/context_numbers_244
|
||||
- data/raw_datasets/context_numbers_245
|
||||
- data/raw_datasets/context_numbers_246
|
||||
- data/raw_datasets/context_numbers_247
|
||||
- data/raw_datasets/context_numbers_248
|
||||
- data/raw_datasets/context_numbers_249
|
||||
- data/raw_datasets/context_numbers_250
|
||||
- data/raw_datasets/context_numbers_251
|
||||
- data/raw_datasets/context_numbers_252
|
||||
- data/raw_datasets/context_numbers_253
|
||||
- data/raw_datasets/context_numbers_254
|
||||
- data/raw_datasets/context_numbers_255
|
||||
- data/raw_datasets/context_numbers_256
|
||||
|
||||
val_ds_names:
|
||||
|
|
@ -300,7 +56,8 @@ val_ds_names:
|
|||
- data/raw_datasets/context_numbers_256
|
||||
|
||||
test_ds_names:
|
||||
- data/raw_datasets/context_numbers_512
|
||||
- data/raw_datasets/context_numbers_1024
|
||||
- data/raw_datasets/context_numbers_2048
|
||||
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_128
|
||||
- data/raw_datasets/context_numbers_256
|
||||
|
|
|
|||
|
|
@ -1,68 +0,0 @@
|
|||
output_dir: "" # just a placeholder
|
||||
bf16: true
|
||||
model_name_or_path: meta-llama/Llama-3.2-1B-Instruct
|
||||
label_names: ["labels"]
|
||||
# eval_on_start: True
|
||||
# eval_strategy: "steps"
|
||||
# eval_steps: 500
|
||||
# save_strategy: "no"
|
||||
# # save_steps: 500
|
||||
# logging_strategy: "steps"
|
||||
# logging_steps: 100
|
||||
# use_liger_kernel: true
|
||||
# remove_unused_columns: false
|
||||
|
||||
# needed to avoid OOM by compute the metrics batch by batch
|
||||
# w/o this the trainer stores logits of all sample in memory...
|
||||
# batch_eval_metrics: true
|
||||
|
||||
per_device_train_batch_size: 32
|
||||
per_device_eval_batch_size: 1
|
||||
max_val_samples_per_ds: 20
|
||||
# optim: schedule_free_adamw
|
||||
learning_rate: 0.00001
|
||||
# lr_scheduler_type: "constant_with_warmup"
|
||||
neftune_noise_alpha: 1
|
||||
weight_decay: 0.1
|
||||
warmup_ratio: 0.05
|
||||
|
||||
# LoRA
|
||||
lora_r: 16
|
||||
lora_dropout: 0.05
|
||||
target_modules:
|
||||
- down_proj
|
||||
- up_proj
|
||||
- gate_proj
|
||||
|
||||
# data
|
||||
train_ds_names:
|
||||
- data/raw_datasets/context_numbers_4
|
||||
- data/raw_datasets/context_numbers_8
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_80
|
||||
- data/raw_datasets/context_numbers_96
|
||||
- data/raw_datasets/context_numbers_112
|
||||
- data/raw_datasets/context_numbers_128
|
||||
- data/raw_datasets/context_numbers_144
|
||||
- data/raw_datasets/context_numbers_160
|
||||
- data/raw_datasets/context_numbers_176
|
||||
- data/raw_datasets/context_numbers_192
|
||||
- data/raw_datasets/context_numbers_208
|
||||
- data/raw_datasets/context_numbers_224
|
||||
- data/raw_datasets/context_numbers_240
|
||||
- data/raw_datasets/context_numbers_256
|
||||
|
||||
val_ds_names:
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_128
|
||||
- data/raw_datasets/context_numbers_256
|
||||
|
||||
test_ds_names:
|
||||
- data/raw_datasets/context_numbers_512
|
||||
- data/raw_datasets/context_numbers_1024
|
||||
- data/raw_datasets/context_numbers_2048
|
||||
|
||||
|
|
@ -64,7 +64,9 @@ val_ds_names:
|
|||
- data/raw_datasets/context_numbers_256
|
||||
|
||||
test_ds_names:
|
||||
- data/raw_datasets/context_numbers_512
|
||||
- data/raw_datasets/context_numbers_1024
|
||||
- data/raw_datasets/context_numbers_2048
|
||||
- data/raw_datasets/context_numbers_16
|
||||
- data/raw_datasets/context_numbers_32
|
||||
- data/raw_datasets/context_numbers_64
|
||||
- data/raw_datasets/context_numbers_128
|
||||
- data/raw_datasets/context_numbers_256
|
||||
- sggetao/PwC
|
||||
|
|
|
|||
|
|
@ -224,7 +224,7 @@ class CtxTrainingArguments:
|
|||
metadata={"help": "Maximum base length for training."},
|
||||
)
|
||||
max_new_tokens: Optional[int] = field(
|
||||
default=2**13,
|
||||
default=2**10,
|
||||
metadata={"help": "Maximum new tokens for generation-based evaluation."},
|
||||
)
|
||||
gen_per_device_eval_batch_size: Optional[int] = field(
|
||||
|
|
@ -255,6 +255,10 @@ class DataArguments:
|
|||
default=5000,
|
||||
metadata={"help": "Maximum number of validation samples per dataset."},
|
||||
)
|
||||
max_test_samples_per_ds: Optional[int] = field(
|
||||
default=1000,
|
||||
metadata={"help": "Maximum number of test samples per dataset."},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -301,7 +301,16 @@ def main(output_dir):
|
|||
|
||||
val_train_indices = np.random.permutation(len(train_ds))[:500]
|
||||
val_ds["train"] = train_ds.select(val_train_indices)
|
||||
test_ds = tokenized_ds.get("test", None)
|
||||
|
||||
test_ds = dict()
|
||||
if "test" in tokenized_ds:
|
||||
for ds_name, ds in tokenized_ds["test"].items():
|
||||
test_ds[ds_name] = ds
|
||||
test_ds_size = len(test_ds[ds_name])
|
||||
test_indices = np.random.permutation(test_ds_size)[
|
||||
: data_args.max_test_samples_per_ds
|
||||
]
|
||||
test_ds[ds_name] = test_ds[ds_name].select(test_indices)
|
||||
|
||||
logger.info(f"train_ds: {train_ds}")
|
||||
logger.info(f"val_ds: {val_ds}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue