From 6c579fa3737129c789784c40d4d88449ae073004 Mon Sep 17 00:00:00 2001 From: 51616 Date: Tue, 7 Jan 2025 16:57:23 +0000 Subject: [PATCH] limit max test sample + config test up to 256 number --- configs/context_numbers_128_new.yaml | 62 ------- configs/context_numbers_256.yaml | 253 +-------------------------- configs/context_numbers_256_new.yaml | 68 ------- configs/pwc_and_ctx_numbers_256.yaml | 8 +- hyperlora/configs.py | 6 +- hyperlora/intx_sft.py | 11 +- 6 files changed, 25 insertions(+), 383 deletions(-) delete mode 100644 configs/context_numbers_128_new.yaml delete mode 100644 configs/context_numbers_256_new.yaml diff --git a/configs/context_numbers_128_new.yaml b/configs/context_numbers_128_new.yaml deleted file mode 100644 index 529e743..0000000 --- a/configs/context_numbers_128_new.yaml +++ /dev/null @@ -1,62 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.1 -warmup_ratio: 0.05 - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_48 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_80 -- data/raw_datasets/context_numbers_96 -- data/raw_datasets/context_numbers_112 -- data/raw_datasets/context_numbers_128 - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_512 -- data/raw_datasets/context_numbers_1024 -- data/raw_datasets/context_numbers_2048 - diff --git a/configs/context_numbers_256.yaml b/configs/context_numbers_256.yaml index 92cba34..0d124f6 100644 --- a/configs/context_numbers_256.yaml +++ b/configs/context_numbers_256.yaml @@ -36,260 +36,16 @@ target_modules: # data train_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_3 - data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_5 -- data/raw_datasets/context_numbers_6 -- data/raw_datasets/context_numbers_7 - data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_9 -- data/raw_datasets/context_numbers_10 -- data/raw_datasets/context_numbers_11 -- data/raw_datasets/context_numbers_12 -- data/raw_datasets/context_numbers_13 -- data/raw_datasets/context_numbers_14 -- data/raw_datasets/context_numbers_15 - data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_17 -- data/raw_datasets/context_numbers_18 -- data/raw_datasets/context_numbers_19 -- data/raw_datasets/context_numbers_20 -- data/raw_datasets/context_numbers_21 -- data/raw_datasets/context_numbers_22 -- data/raw_datasets/context_numbers_23 -- data/raw_datasets/context_numbers_24 -- data/raw_datasets/context_numbers_25 -- data/raw_datasets/context_numbers_26 -- data/raw_datasets/context_numbers_27 -- data/raw_datasets/context_numbers_28 -- data/raw_datasets/context_numbers_29 -- data/raw_datasets/context_numbers_30 -- data/raw_datasets/context_numbers_31 - data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_33 -- data/raw_datasets/context_numbers_34 -- data/raw_datasets/context_numbers_35 -- data/raw_datasets/context_numbers_36 -- data/raw_datasets/context_numbers_37 -- data/raw_datasets/context_numbers_38 -- data/raw_datasets/context_numbers_39 -- data/raw_datasets/context_numbers_40 -- data/raw_datasets/context_numbers_41 -- data/raw_datasets/context_numbers_42 -- data/raw_datasets/context_numbers_43 -- data/raw_datasets/context_numbers_44 -- data/raw_datasets/context_numbers_45 -- data/raw_datasets/context_numbers_46 -- data/raw_datasets/context_numbers_47 -- data/raw_datasets/context_numbers_48 -- data/raw_datasets/context_numbers_49 -- data/raw_datasets/context_numbers_50 -- data/raw_datasets/context_numbers_51 -- data/raw_datasets/context_numbers_52 -- data/raw_datasets/context_numbers_53 -- data/raw_datasets/context_numbers_54 -- data/raw_datasets/context_numbers_55 -- data/raw_datasets/context_numbers_56 -- data/raw_datasets/context_numbers_57 -- data/raw_datasets/context_numbers_58 -- data/raw_datasets/context_numbers_59 -- data/raw_datasets/context_numbers_60 -- data/raw_datasets/context_numbers_61 -- data/raw_datasets/context_numbers_62 -- data/raw_datasets/context_numbers_63 - data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_65 -- data/raw_datasets/context_numbers_66 -- data/raw_datasets/context_numbers_67 -- data/raw_datasets/context_numbers_68 -- data/raw_datasets/context_numbers_69 -- data/raw_datasets/context_numbers_70 -- data/raw_datasets/context_numbers_71 -- data/raw_datasets/context_numbers_72 -- data/raw_datasets/context_numbers_73 -- data/raw_datasets/context_numbers_74 -- data/raw_datasets/context_numbers_75 -- data/raw_datasets/context_numbers_76 -- data/raw_datasets/context_numbers_77 -- data/raw_datasets/context_numbers_78 -- data/raw_datasets/context_numbers_79 -- data/raw_datasets/context_numbers_80 -- data/raw_datasets/context_numbers_81 -- data/raw_datasets/context_numbers_82 -- data/raw_datasets/context_numbers_83 -- data/raw_datasets/context_numbers_84 -- data/raw_datasets/context_numbers_85 -- data/raw_datasets/context_numbers_86 -- data/raw_datasets/context_numbers_87 -- data/raw_datasets/context_numbers_88 -- data/raw_datasets/context_numbers_89 -- data/raw_datasets/context_numbers_90 -- data/raw_datasets/context_numbers_91 -- data/raw_datasets/context_numbers_92 -- data/raw_datasets/context_numbers_93 -- data/raw_datasets/context_numbers_94 -- data/raw_datasets/context_numbers_95 - data/raw_datasets/context_numbers_96 -- data/raw_datasets/context_numbers_97 -- data/raw_datasets/context_numbers_98 -- data/raw_datasets/context_numbers_99 -- data/raw_datasets/context_numbers_100 -- data/raw_datasets/context_numbers_101 -- data/raw_datasets/context_numbers_102 -- data/raw_datasets/context_numbers_103 -- data/raw_datasets/context_numbers_104 -- data/raw_datasets/context_numbers_105 -- data/raw_datasets/context_numbers_106 -- data/raw_datasets/context_numbers_107 -- data/raw_datasets/context_numbers_108 -- data/raw_datasets/context_numbers_109 -- data/raw_datasets/context_numbers_110 -- data/raw_datasets/context_numbers_111 -- data/raw_datasets/context_numbers_112 -- data/raw_datasets/context_numbers_113 -- data/raw_datasets/context_numbers_114 -- data/raw_datasets/context_numbers_115 -- data/raw_datasets/context_numbers_116 -- data/raw_datasets/context_numbers_117 -- data/raw_datasets/context_numbers_118 -- data/raw_datasets/context_numbers_119 -- data/raw_datasets/context_numbers_120 -- data/raw_datasets/context_numbers_121 -- data/raw_datasets/context_numbers_122 -- data/raw_datasets/context_numbers_123 -- data/raw_datasets/context_numbers_124 -- data/raw_datasets/context_numbers_125 -- data/raw_datasets/context_numbers_126 -- data/raw_datasets/context_numbers_127 - data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_129 -- data/raw_datasets/context_numbers_130 -- data/raw_datasets/context_numbers_131 -- data/raw_datasets/context_numbers_132 -- data/raw_datasets/context_numbers_133 -- data/raw_datasets/context_numbers_134 -- data/raw_datasets/context_numbers_135 -- data/raw_datasets/context_numbers_136 -- data/raw_datasets/context_numbers_137 -- data/raw_datasets/context_numbers_138 -- data/raw_datasets/context_numbers_139 -- data/raw_datasets/context_numbers_140 -- data/raw_datasets/context_numbers_141 -- data/raw_datasets/context_numbers_142 -- data/raw_datasets/context_numbers_143 -- data/raw_datasets/context_numbers_144 -- data/raw_datasets/context_numbers_145 -- data/raw_datasets/context_numbers_146 -- data/raw_datasets/context_numbers_147 -- data/raw_datasets/context_numbers_148 -- data/raw_datasets/context_numbers_149 -- data/raw_datasets/context_numbers_150 -- data/raw_datasets/context_numbers_151 -- data/raw_datasets/context_numbers_152 -- data/raw_datasets/context_numbers_153 -- data/raw_datasets/context_numbers_154 -- data/raw_datasets/context_numbers_155 -- data/raw_datasets/context_numbers_156 -- data/raw_datasets/context_numbers_157 -- data/raw_datasets/context_numbers_158 -- data/raw_datasets/context_numbers_159 - data/raw_datasets/context_numbers_160 -- data/raw_datasets/context_numbers_161 -- data/raw_datasets/context_numbers_162 -- data/raw_datasets/context_numbers_163 -- data/raw_datasets/context_numbers_164 -- data/raw_datasets/context_numbers_165 -- data/raw_datasets/context_numbers_166 -- data/raw_datasets/context_numbers_167 -- data/raw_datasets/context_numbers_168 -- data/raw_datasets/context_numbers_169 -- data/raw_datasets/context_numbers_170 -- data/raw_datasets/context_numbers_171 -- data/raw_datasets/context_numbers_172 -- data/raw_datasets/context_numbers_173 -- data/raw_datasets/context_numbers_174 -- data/raw_datasets/context_numbers_175 -- data/raw_datasets/context_numbers_176 -- data/raw_datasets/context_numbers_177 -- data/raw_datasets/context_numbers_178 -- data/raw_datasets/context_numbers_179 -- data/raw_datasets/context_numbers_180 -- data/raw_datasets/context_numbers_181 -- data/raw_datasets/context_numbers_182 -- data/raw_datasets/context_numbers_183 -- data/raw_datasets/context_numbers_184 -- data/raw_datasets/context_numbers_185 -- data/raw_datasets/context_numbers_186 -- data/raw_datasets/context_numbers_187 -- data/raw_datasets/context_numbers_188 -- data/raw_datasets/context_numbers_189 -- data/raw_datasets/context_numbers_190 -- data/raw_datasets/context_numbers_191 - data/raw_datasets/context_numbers_192 -- data/raw_datasets/context_numbers_193 -- data/raw_datasets/context_numbers_194 -- data/raw_datasets/context_numbers_195 -- data/raw_datasets/context_numbers_196 -- data/raw_datasets/context_numbers_197 -- data/raw_datasets/context_numbers_198 -- data/raw_datasets/context_numbers_199 -- data/raw_datasets/context_numbers_200 -- data/raw_datasets/context_numbers_201 -- data/raw_datasets/context_numbers_202 -- data/raw_datasets/context_numbers_203 -- data/raw_datasets/context_numbers_204 -- data/raw_datasets/context_numbers_205 -- data/raw_datasets/context_numbers_206 -- data/raw_datasets/context_numbers_207 -- data/raw_datasets/context_numbers_208 -- data/raw_datasets/context_numbers_209 -- data/raw_datasets/context_numbers_210 -- data/raw_datasets/context_numbers_211 -- data/raw_datasets/context_numbers_212 -- data/raw_datasets/context_numbers_213 -- data/raw_datasets/context_numbers_214 -- data/raw_datasets/context_numbers_215 -- data/raw_datasets/context_numbers_216 -- data/raw_datasets/context_numbers_217 -- data/raw_datasets/context_numbers_218 -- data/raw_datasets/context_numbers_219 -- data/raw_datasets/context_numbers_220 -- data/raw_datasets/context_numbers_221 -- data/raw_datasets/context_numbers_222 -- data/raw_datasets/context_numbers_223 - data/raw_datasets/context_numbers_224 -- data/raw_datasets/context_numbers_225 -- data/raw_datasets/context_numbers_226 -- data/raw_datasets/context_numbers_227 -- data/raw_datasets/context_numbers_228 -- data/raw_datasets/context_numbers_229 -- data/raw_datasets/context_numbers_230 -- data/raw_datasets/context_numbers_231 -- data/raw_datasets/context_numbers_232 -- data/raw_datasets/context_numbers_233 -- data/raw_datasets/context_numbers_234 -- data/raw_datasets/context_numbers_235 -- data/raw_datasets/context_numbers_236 -- data/raw_datasets/context_numbers_237 -- data/raw_datasets/context_numbers_238 -- data/raw_datasets/context_numbers_239 -- data/raw_datasets/context_numbers_240 -- data/raw_datasets/context_numbers_241 -- data/raw_datasets/context_numbers_242 -- data/raw_datasets/context_numbers_243 -- data/raw_datasets/context_numbers_244 -- data/raw_datasets/context_numbers_245 -- data/raw_datasets/context_numbers_246 -- data/raw_datasets/context_numbers_247 -- data/raw_datasets/context_numbers_248 -- data/raw_datasets/context_numbers_249 -- data/raw_datasets/context_numbers_250 -- data/raw_datasets/context_numbers_251 -- data/raw_datasets/context_numbers_252 -- data/raw_datasets/context_numbers_253 -- data/raw_datasets/context_numbers_254 -- data/raw_datasets/context_numbers_255 - data/raw_datasets/context_numbers_256 val_ds_names: @@ -300,7 +56,8 @@ val_ds_names: - data/raw_datasets/context_numbers_256 test_ds_names: -- data/raw_datasets/context_numbers_512 -- data/raw_datasets/context_numbers_1024 -- data/raw_datasets/context_numbers_2048 - +- data/raw_datasets/context_numbers_16 +- data/raw_datasets/context_numbers_32 +- data/raw_datasets/context_numbers_64 +- data/raw_datasets/context_numbers_128 +- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_256_new.yaml b/configs/context_numbers_256_new.yaml deleted file mode 100644 index 03e1e2f..0000000 --- a/configs/context_numbers_256_new.yaml +++ /dev/null @@ -1,68 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.1 -warmup_ratio: 0.05 - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_80 -- data/raw_datasets/context_numbers_96 -- data/raw_datasets/context_numbers_112 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_144 -- data/raw_datasets/context_numbers_160 -- data/raw_datasets/context_numbers_176 -- data/raw_datasets/context_numbers_192 -- data/raw_datasets/context_numbers_208 -- data/raw_datasets/context_numbers_224 -- data/raw_datasets/context_numbers_240 -- data/raw_datasets/context_numbers_256 - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_512 -- data/raw_datasets/context_numbers_1024 -- data/raw_datasets/context_numbers_2048 - diff --git a/configs/pwc_and_ctx_numbers_256.yaml b/configs/pwc_and_ctx_numbers_256.yaml index fd96c31..cf14a24 100644 --- a/configs/pwc_and_ctx_numbers_256.yaml +++ b/configs/pwc_and_ctx_numbers_256.yaml @@ -64,7 +64,9 @@ val_ds_names: - data/raw_datasets/context_numbers_256 test_ds_names: -- data/raw_datasets/context_numbers_512 -- data/raw_datasets/context_numbers_1024 -- data/raw_datasets/context_numbers_2048 +- data/raw_datasets/context_numbers_16 +- data/raw_datasets/context_numbers_32 +- data/raw_datasets/context_numbers_64 +- data/raw_datasets/context_numbers_128 +- data/raw_datasets/context_numbers_256 - sggetao/PwC diff --git a/hyperlora/configs.py b/hyperlora/configs.py index c137a38..ead6daa 100644 --- a/hyperlora/configs.py +++ b/hyperlora/configs.py @@ -224,7 +224,7 @@ class CtxTrainingArguments: metadata={"help": "Maximum base length for training."}, ) max_new_tokens: Optional[int] = field( - default=2**13, + default=2**10, metadata={"help": "Maximum new tokens for generation-based evaluation."}, ) gen_per_device_eval_batch_size: Optional[int] = field( @@ -255,6 +255,10 @@ class DataArguments: default=5000, metadata={"help": "Maximum number of validation samples per dataset."}, ) + max_test_samples_per_ds: Optional[int] = field( + default=1000, + metadata={"help": "Maximum number of test samples per dataset."}, + ) @dataclass diff --git a/hyperlora/intx_sft.py b/hyperlora/intx_sft.py index e938bf9..6e23ab4 100644 --- a/hyperlora/intx_sft.py +++ b/hyperlora/intx_sft.py @@ -301,7 +301,16 @@ def main(output_dir): val_train_indices = np.random.permutation(len(train_ds))[:500] val_ds["train"] = train_ds.select(val_train_indices) - test_ds = tokenized_ds.get("test", None) + + test_ds = dict() + if "test" in tokenized_ds: + for ds_name, ds in tokenized_ds["test"].items(): + test_ds[ds_name] = ds + test_ds_size = len(test_ds[ds_name]) + test_indices = np.random.permutation(test_ds_size)[ + : data_args.max_test_samples_per_ds + ] + test_ds[ds_name] = test_ds[ds_name].select(test_indices) logger.info(f"train_ds: {train_ds}") logger.info(f"val_ds: {val_ds}")