diff --git a/batch_gemma_llama_vision.sh b/batch_gemma_llama_vision.sh index 03ff5c8..396511a 100755 --- a/batch_gemma_llama_vision.sh +++ b/batch_gemma_llama_vision.sh @@ -20,11 +20,11 @@ conda activate /home/rujikorn_sakana_ai/.conda/envs/llm-modulator accelerate launch --num_processes=8 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ --gpu_ids all --main_process_port 29554 intx_sft.py configs/pretrain_all.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=4 \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=8 \ --gradient_accumulation_steps=4 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj --num_blocks=1 --num_self_attends_per_block=16 \ ---self_attention_widening_factor=1 --eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ +--target_modules=down_proj,up_proj --num_blocks=1 --num_self_attends_per_block=32 \ +--self_attention_widening_factor=4 --eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 \ +--neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=1024 \ --load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ --add_repeat_prompt=False --ctx_encoder_model_name_or_path=meta-llama/Llama-3.2-11B-Vision-Instruct diff --git a/batch_llama_3_8b_1k_latent.sh b/batch_llama_3_8b_1k_latent.sh index 374af09..1969850 100755 --- a/batch_llama_3_8b_1k_latent.sh +++ b/batch_llama_3_8b_1k_latent.sh @@ -20,7 +20,7 @@ conda activate /home/rujikorn_sakana_ai/.conda/envs/llm-modulator accelerate launch --num_processes=8 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ --gpu_ids all --main_process_port 29555 intx_sft.py configs/pretrain_all.yaml \ ---model_name_or_path=meta-llama/Llama-3.1-8B-Instruct --num_train_epochs=5.1 --per_device_train_batch_size=4 \ +--model_name_or_path=meta-llama/Llama-3.1-8B-Instruct --num_train_epochs=5.1 --per_device_train_batch_size=8 \ --gradient_accumulation_steps=4 --per_device_eval_batch_size=4 --exp_setup=hyper_lora --aggregator_type=perceiver \ --target_modules=down_proj,up_proj --num_blocks=1 --num_self_attends_per_block=32 \ --self_attention_widening_factor=4 --eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 \ diff --git a/install.sh b/install.sh index 6fee47a..d33086c 100644 --- a/install.sh +++ b/install.sh @@ -1,2 +1,3 @@ pip install -e . pip install flash-attn --no-build-isolation +# huggingface-cli login diff --git a/pyproject.toml b/pyproject.toml index a1fa627..2a7ff69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ "flask", "pandas", "plotly", + "rouge-score", ] [build-system] diff --git a/requirements.txt b/requirements.txt index 4e0ffcd..472156f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ einops jaxtyping liger-kernel +rouge-score transformers==4.46.3 deepspeed==0.15.4 # curretly 0.16.2 accelerate==1.2.1