mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
transformers ver + squad only example
This commit is contained in:
51616
parent
3732f0860b
commit
36d78c827c
4 changed files with 52 additions and 7 deletions
|
|
@ -31,11 +31,11 @@ print(model.decode(outputs))
|
|||
## 🏋️ Training
|
||||
### 🔢 HyperLoRA w/ context_numbers_10
|
||||
```bash
|
||||
WANDB_MODE=disabled run uv run intx_sft.py configs/context_numbers_10.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=1 --per_device_train_batch_size=64 --gradient_accumulation_steps=1 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=2 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=False --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --use_sequence_packing=True --max_packed_inp_len=8000 --max_packed_ctx_len=16000
|
||||
WANDB_MODE=disabled uv run intx_sft.py configs/context_numbers_10.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=1 --per_device_train_batch_size=64 --gradient_accumulation_steps=1 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=2 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=False --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --use_sequence_packing=True --max_packed_inp_len=8000 --max_packed_ctx_len=16000
|
||||
```
|
||||
### Squad + Hotpot training (for testing/debugging)
|
||||
```bash
|
||||
WANDB_MODE=disabled run uv run intx_sft.py configs/squad_hotpot_qa.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --metric_for_best_model=eval_pwc_loss --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=16000 --max_packed_ctx_len=32000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1
|
||||
WANDB_MODE=disabled uv run intx_sft.py configs/squad.yaml --model_name_or_path=google/gemma-3-1b-it --num_train_epochs=5 --per_device_train_batch_size=64 --gradient_accumulation_steps=8 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --num_self_attends_per_block=8 --num_latent_factor=1 --num_pre_head_layers=1 --lora_r=8 --eval_steps=1000 --save_steps=1000 --learning_rate=4e-5 --lora_dropout=0.0 --neftune_noise_alpha=5 --use_light_weight_lora=False --load_best_model_at_end=False --metric_for_best_model=eval_pwc_loss --add_negative_prompt=False --add_repeat_prompt=False --use_sequence_packing=True --max_packed_inp_len=16000 --max_packed_ctx_len=32000 --per_rank_gen=True --per_layer_processing=True --gen_lora_l1_reg_coef=0.1 --logging_steps=10
|
||||
```
|
||||
|
||||
### HyperLoRA w/ self-gen 3 mini
|
||||
|
|
|
|||
45
configs/squad.yaml
Normal file
45
configs/squad.yaml
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
output_dir: "" # just a placeholder
|
||||
bf16: true
|
||||
model_name_or_path: google/gemma-3-1b-it
|
||||
label_names: ["labels"]
|
||||
# eval_on_start: True
|
||||
# eval_strategy: "steps"
|
||||
# eval_steps: 500
|
||||
# save_strategy: "no"
|
||||
# # save_steps: 500
|
||||
# logging_strategy: "steps"
|
||||
# logging_steps: 100
|
||||
# use_liger_kernel: true
|
||||
# remove_unused_columns: false
|
||||
|
||||
# needed to avoid OOM by compute the metrics batch by batch
|
||||
# w/o this the trainer stores logits of all sample in memory...
|
||||
# batch_eval_metrics: true
|
||||
|
||||
per_device_train_batch_size: 8
|
||||
per_device_eval_batch_size: 8
|
||||
max_val_samples_per_ds: 1000
|
||||
# optim: schedule_free_adamw
|
||||
|
||||
learning_rate: 0.00004
|
||||
# lr_scheduler_type: "constant_with_warmup"
|
||||
neftune_noise_alpha: 1
|
||||
weight_decay: 0.01
|
||||
|
||||
warmup_steps: 100
|
||||
|
||||
dataloader_prefetch_factor: 16
|
||||
dataloader_num_workers: 8
|
||||
|
||||
# LoRA
|
||||
lora_r: 8
|
||||
lora_dropout: 0.0
|
||||
target_modules:
|
||||
- down_proj
|
||||
|
||||
# data
|
||||
train_ds_names:
|
||||
- squad
|
||||
|
||||
val_ds_names:
|
||||
- squad
|
||||
|
|
@ -6,7 +6,7 @@ description = ""
|
|||
readme = "README.md"
|
||||
requires-python = ">= 3.10"
|
||||
dependencies = [
|
||||
"transformers==4.51.3",
|
||||
"transformers==4.52.4",
|
||||
"deepspeed==0.17.1",
|
||||
"accelerate==1.6.0",
|
||||
"datasets==3.6.0",
|
||||
|
|
|
|||
8
uv.lock
generated
8
uv.lock
generated
|
|
@ -726,7 +726,7 @@ requires-dist = [
|
|||
{ name = "tensorboard" },
|
||||
{ name = "tensorboardx" },
|
||||
{ name = "torchmetrics" },
|
||||
{ name = "transformers", specifier = "==4.51.3" },
|
||||
{ name = "transformers", specifier = "==4.52.4" },
|
||||
{ name = "vllm", specifier = "==0.8.5" },
|
||||
{ name = "wandb" },
|
||||
]
|
||||
|
|
@ -5025,7 +5025,7 @@ wheels = [
|
|||
|
||||
[[package]]
|
||||
name = "transformers"
|
||||
version = "4.51.3"
|
||||
version = "4.52.4"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "filelock" },
|
||||
|
|
@ -5039,9 +5039,9 @@ dependencies = [
|
|||
{ name = "tokenizers" },
|
||||
{ name = "tqdm" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f1/11/7414d5bc07690002ce4d7553602107bf969af85144bbd02830f9fb471236/transformers-4.51.3.tar.gz", hash = "sha256:e292fcab3990c6defe6328f0f7d2004283ca81a7a07b2de9a46d67fd81ea1409", size = 8941266, upload-time = "2025-04-14T08:15:00.485Z" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/da/a9/275037087f9d846580b02f2d7cae0e0a6955d46f84583d0151d6227bd416/transformers-4.52.4.tar.gz", hash = "sha256:aff3764441c1adc192a08dba49740d3cbbcb72d850586075aed6bd89b98203e6", size = 8945376, upload-time = "2025-05-30T09:17:17.947Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/b6/5257d04ae327b44db31f15cce39e6020cc986333c715660b1315a9724d82/transformers-4.51.3-py3-none-any.whl", hash = "sha256:fd3279633ceb2b777013234bbf0b4f5c2d23c4626b05497691f00cfda55e8a83", size = 10383940, upload-time = "2025-04-14T08:13:43.023Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/f2/25b27b396af03d5b64e61976b14f7209e2939e9e806c10749b6d277c273e/transformers-4.52.4-py3-none-any.whl", hash = "sha256:203f5c19416d5877e36e88633943761719538a25d9775977a24fe77a1e5adfc7", size = 10460375, upload-time = "2025-05-30T09:17:14.477Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue