apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

load data from disk then convert to iterable

Browse source
This commit is contained in:
51616 2025-05-09 13:11:34 +00:00
parent bd61787486
commit dde5f5cb4b
5 changed files with 88 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

2
scripts/fw_qa_3/gemma_per_rank_fac2_per_layer.sh
View file

@ -21,7 +21,7 @@ conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora
accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \
--gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl_3.yaml \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \
--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=32 \
--gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \
--target_modules=down_proj \
--num_self_attends_per_block=8 --num_latent_factor=2 \