apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

cmd readme

Browse source
This commit is contained in:
51616 2025-06-03 14:29:04 +00:00
parent 1cb81b59ec
commit 5d793e329b
1 changed files with 8 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

16
README.md
View file

@ -33,11 +33,13 @@ print(model.decode(outputs))
```bash
WANDB_MODE=disabled uv run python intx_sft.py configs/context_numbers_10.yaml --model_name_or_path=meta-llama/Llama-3.2-1B-Instruct --num_train_epochs=100 --per_device_train_batch_size=64 --per_device_eval_batch_size=64 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj --per_rank_gen=True --per_layer_processing=True --decoder_depth=2 --seed=1 --gen_lora_l1_reg_coef=0 --use_token_mixing=False --lora_r=8 --bf16=True --tf32=True --dataloader_num_workers=8 --dataloader_prefetch_factor=8
```
<!-- # TODO: sequence packing remove cols? -->
### PwC + Hotpot training (for testing/debugging)
```bash
uv run intx_sft.py configs/pwc_hotpot_qa.yaml \
WANDB_MODE=disabled uv run intx_sft.py configs/pwc_hotpot_qa.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=32 \
--gradient_accumulation_steps=1 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \
--gradient_accumulation_steps=1 --per_device_eval_batch_size=32 --exp_setup=hyper_lora \
--aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=4 --num_latent_factor=2 \
--lora_r=8 \
@ -116,18 +118,16 @@ run python intx_sft.py configs/...yaml ... --from_pretrained_checkpoint=train_ou
LongBench
```bash
# generative
run python src/ctx_to_lora/eval.py --checkpoint_path train_outputs/runs/.../pytorch_model.bin --datasets negative_nq triviaqa_retrieved squad longbench_e --split test
run python run_eval.py --checkpoint_path train_outputs/runs/.../pytorch_model.bin --datasets negative_nq triviaqa_retrieved squad longbench_e --split test
# hypernet checkpoint
uv run python eval.py --checkpoint_path train_outputs/runs/May08_13-56-31_slurm0-a3nodeset-5_59383_906acb28/checkpoint-105000/pytorch_model.bin --datasets negative_nq triviaqa_retrieved squad longbench_e --split test
uv run python run_eval.py --checkpoint_path train_outputs/runs/May08_13-56-31_slurm0-a3nodeset-5_59383_906acb28/checkpoint-105000/pytorch_model.bin --datasets negative_nq triviaqa_retrieved squad longbench_e --split test
# base model
uv run python eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench_e --split test
uv run python run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench_e --split test
# base model w/o context
run uv run python eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench_e --split test --remove_context
# run python src/ctx_to_lora/eval.py --checkpoint_path train_outputs/runs/May08_13-56-31_slurm0-a3nodeset-5_59383_906acb28/checkpoint-105000/pytorch_model.bin
run uv run python run_eval.py --model_name_or_path google/gemma-2-2b-it --datasets longbench_e --split test --remove_context
# # benchmark
# cd LongBench/LongBench