mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
65 lines
1.4 KiB
YAML
65 lines
1.4 KiB
YAML
output_dir: "" # just a placeholder
|
|
bf16: true
|
|
model_name_or_path: google/gemma-2-2b-it
|
|
label_names: ["labels"]
|
|
# eval_on_start: True
|
|
# eval_strategy: "steps"
|
|
# eval_steps: 500
|
|
# save_strategy: "no"
|
|
# # save_steps: 500
|
|
# logging_strategy: "steps"
|
|
# logging_steps: 100
|
|
# use_liger_kernel: true
|
|
# remove_unused_columns: false
|
|
|
|
# needed to avoid OOM by compute the metrics batch by batch
|
|
# w/o this the trainer stores logits of all sample in memory...
|
|
# batch_eval_metrics: true
|
|
|
|
per_device_train_batch_size: 8
|
|
per_device_eval_batch_size: 8
|
|
max_val_samples_per_ds: 1000
|
|
# optim: schedule_free_adamw
|
|
|
|
learning_rate: 0.00004
|
|
# lr_scheduler_type: "constant_with_warmup"
|
|
neftune_noise_alpha: 5
|
|
weight_decay: 0.01
|
|
#
|
|
warmup_steps: 100
|
|
|
|
dataloader_prefetch_factor: 8
|
|
dataloader_num_workers: 8
|
|
# LoRA
|
|
lora_r: 8
|
|
lora_dropout: 0.0
|
|
target_modules:
|
|
- down_proj
|
|
# data
|
|
train_ds_names:
|
|
- fw_qa_3_small # ~ 20M
|
|
- ctx_qa # 300k
|
|
- pwc # 240k
|
|
- hotpot_qa # 90k
|
|
- squad # 90k
|
|
- drop # 77k
|
|
- narrativeqa # 40k
|
|
- quoref # 11k
|
|
- ropes # 11k
|
|
- synthetic_convqa # 40k
|
|
|
|
val_ds_names:
|
|
- fw_qa_3_pretrain
|
|
- self_gen/google/gemma-2-2b-it/fw_qa_3_small
|
|
- self_gen/google/gemma-2-2b-it/ctx_qa
|
|
- self_gen/google/gemma-2-2b-it/pwc
|
|
- self_gen/google/gemma-2-2b-it/hotpot_qa
|
|
- self_gen/google/gemma-2-2b-it/squad
|
|
- fw_qa_3
|
|
- ctx_qa
|
|
- pwc
|
|
- hotpot_qa
|
|
- squad
|
|
|
|
load_best_model_at_end: false
|
|
metric_for_best_model: eval_fw_qa_3_pretrain_loss
|