From a9a164e2de25780f3e59e477eee09cb72d2721ca Mon Sep 17 00:00:00 2001 From: 51616 Date: Fri, 6 Jun 2025 04:56:45 +0000 Subject: [PATCH] clean up configs + scripts + chat_templates (+ %generation% tag) --- chat_templates/google/gemma-2-2b-it.jinja | 2 + .../meta-llama/Llama-3.1-8B-Instruct.jinja | 103 ---------------- .../Llama-3.2-11B-Vision-Instruct.jinja | 110 ------------------ .../meta-llama/Llama-3.2-1B-Instruct.jinja | 83 ------------- .../meta-llama/Llama-3.2-3B-Instruct.jinja | 83 ------------- .../Llama-3.2-90B-Vision-Instruct.jinja | 110 ------------------ .../meta-llama/Llama-3.3-70B-Instruct.jinja | 104 ----------------- configs/context_numbers_10.yaml | 68 ----------- configs/context_numbers_128.yaml | 65 ----------- configs/context_numbers_128_only.yaml | 56 --------- configs/context_numbers_256.yaml | 66 ----------- configs/context_numbers_256_only.yaml | 56 --------- configs/context_numbers_32.yaml | 63 ---------- configs/context_numbers_32_only.yaml | 55 --------- configs/context_numbers_64.yaml | 64 ---------- configs/context_numbers_64_only.yaml | 56 --------- configs/context_numbers_debug.yaml | 47 -------- configs/context_numbers_easy.yaml | 56 --------- configs/default.yaml | 35 ------ configs/fw_qa.yaml | 44 ------- configs/fw_qa_ctx_qa.yaml | 50 -------- configs/fw_qa_large_ctx_pwc_hotpot.yaml | 55 --------- ...otpot_qa.yaml => fw_qa_pretrain_mini.yaml} | 19 ++- ....yaml => fw_qa_pretrain_mini_and_pwc.yaml} | 29 ++--- .../{gsm8k.yaml => fw_qa_pretrain_small.yaml} | 16 ++- ...yaml => fw_qa_pretrain_small_and_pwc.yaml} | 19 +-- configs/fw_qa_pwc_hotpot.yaml | 51 -------- configs/fw_qa_tiny.yaml | 44 ------- configs/hotpot_qa.yaml | 45 ------- configs/math_and_code.yaml | 50 -------- configs/pretrain_all.yaml | 62 ---------- configs/pretrain_all_2.yaml | 61 ---------- configs/pretrain_all_xl.yaml | 61 ---------- configs/pretrain_all_xl_3_medium.yaml | 60 ---------- configs/pretrain_all_xl_3_mini.yaml | 60 ---------- configs/pretrain_all_xl_and_sum.yaml | 63 ---------- configs/pwc.yaml | 45 ------- configs/pwc_and_ctx_numbers_256.yaml | 73 ------------ ...in_all_xl_3.yaml => self_gen_3_small.yaml} | 28 +++-- .../ablate_arch/gemma_xl_per_rank_gen_fac2.sh | 33 ------ .../gemma_xl_per_rank_gen_fac2_neft1.sh | 32 ----- ..._per_rank_gen_fac2_per_layer_processing.sh | 34 ------ ...k_gen_fac2_per_layer_processing_4e-5_lr.sh | 34 ------ ...gen_fac2_per_layer_processing_decoder_2.sh | 35 ------ ..._rank_gen_fac2_per_layer_processing_r16.sh | 34 ------ .../ablate_arch/gemma_xl_per_rank_gen_fac4.sh | 32 ----- ..._per_rank_gen_fac4_per_layer_processing.sh | 33 ------ ...gen_fac4_per_layer_processing_decoder_2.sh | 33 ------ ...nk_gen_fac4_per_layer_processing_no_reg.sh | 33 ------ ..._fac4_per_layer_processing_token_mixing.sh | 33 ------ .../fw_qa_3/gemma_per_rank_fac2_per_layer.sh | 34 ------ .../medium_gemma_per_rank_fac2_per_layer.sh | 33 ------ .../gemma_pretrain.sh} | 18 +-- .../gemma_pretrain_and_pwc.sh} | 17 +-- scripts/repeat_text/gemma_llama_xl.sh | 31 ----- .../gemma_llama_xl.sh | 31 ----- .../gemma_llama_xl_down_ln.sh | 31 ----- .../gemma_llama_xl_ln_only.sh | 31 ----- .../gemma_llama_xl_up_down.sh | 31 ----- .../gemma_llama_xl_up_down_ln.sh | 31 ----- .../gemma_llama_2.sh | 31 ----- .../gemma_llama_xl.sh | 31 ----- .../gemma_llama_xl_per_rank_gen.sh | 31 ----- ...emma_llama_xl_per_rank_gen_latent_fac_1.sh | 32 ----- ...lama_xl_per_rank_gen_latent_fac_1_llama.sh | 33 ------ ...llama_xl_per_rank_gen_latent_fac_1_r_16.sh | 32 ----- .../gemma_llama_xl_r_16.sh | 31 ----- .../gemma_llama_xl_r_16_ln.sh | 31 ----- .../gemma_llama_xl_up_down.sh | 31 ----- .../gemma_llama_xl.sh | 31 ----- .../gemma_llama_xl_down_ln.sh | 31 ----- .../gemma_llama_xl_ln_only.sh | 31 ----- .../gemma_llama_xl_up_down.sh | 31 ----- .../gemma_llama_xl_up_down_ln.sh | 31 ----- ...rank_fac2_per_layer_4_gpus_gen_lora_reg.sh | 0 ...c2_per_layer_4_gpus_gen_lora_reg_repeat.sh | 0 76 files changed, 72 insertions(+), 3237 deletions(-) delete mode 100644 chat_templates/meta-llama/Llama-3.1-8B-Instruct.jinja delete mode 100644 chat_templates/meta-llama/Llama-3.2-11B-Vision-Instruct.jinja delete mode 100644 chat_templates/meta-llama/Llama-3.2-1B-Instruct.jinja delete mode 100644 chat_templates/meta-llama/Llama-3.2-3B-Instruct.jinja delete mode 100644 chat_templates/meta-llama/Llama-3.2-90B-Vision-Instruct.jinja delete mode 100644 chat_templates/meta-llama/Llama-3.3-70B-Instruct.jinja delete mode 100644 configs/context_numbers_10.yaml delete mode 100644 configs/context_numbers_128.yaml delete mode 100644 configs/context_numbers_128_only.yaml delete mode 100644 configs/context_numbers_256.yaml delete mode 100644 configs/context_numbers_256_only.yaml delete mode 100644 configs/context_numbers_32.yaml delete mode 100644 configs/context_numbers_32_only.yaml delete mode 100644 configs/context_numbers_64.yaml delete mode 100644 configs/context_numbers_64_only.yaml delete mode 100644 configs/context_numbers_debug.yaml delete mode 100644 configs/context_numbers_easy.yaml delete mode 100644 configs/default.yaml delete mode 100644 configs/fw_qa.yaml delete mode 100644 configs/fw_qa_ctx_qa.yaml delete mode 100644 configs/fw_qa_large_ctx_pwc_hotpot.yaml rename configs/{pwc_hotpot_qa.yaml => fw_qa_pretrain_mini.yaml} (83%) rename configs/{fw_qa_large_ctx_pwc_hotpot_squad.yaml => fw_qa_pretrain_mini_and_pwc.yaml} (75%) rename configs/{gsm8k.yaml => fw_qa_pretrain_small.yaml} (82%) rename configs/{openmathintx.yaml => fw_qa_pretrain_small_and_pwc.yaml} (75%) delete mode 100644 configs/fw_qa_pwc_hotpot.yaml delete mode 100644 configs/fw_qa_tiny.yaml delete mode 100644 configs/hotpot_qa.yaml delete mode 100644 configs/math_and_code.yaml delete mode 100644 configs/pretrain_all.yaml delete mode 100644 configs/pretrain_all_2.yaml delete mode 100644 configs/pretrain_all_xl.yaml delete mode 100644 configs/pretrain_all_xl_3_medium.yaml delete mode 100644 configs/pretrain_all_xl_3_mini.yaml delete mode 100644 configs/pretrain_all_xl_and_sum.yaml delete mode 100644 configs/pwc.yaml delete mode 100644 configs/pwc_and_ctx_numbers_256.yaml rename configs/{pretrain_all_xl_3.yaml => self_gen_3_small.yaml} (55%) delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac2.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_neft1.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_4e-5_lr.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_decoder_2.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_r16.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac4.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_decoder_2.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_no_reg.sh delete mode 100644 scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_token_mixing.sh delete mode 100644 scripts/fw_qa_3/gemma_per_rank_fac2_per_layer.sh delete mode 100644 scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer.sh rename scripts/{fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus.sh => fw_qa_pretrain/gemma_pretrain.sh} (56%) rename scripts/{fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh => fw_qa_pretrain/gemma_pretrain_and_pwc.sh} (62%) delete mode 100644 scripts/repeat_text/gemma_llama_xl.sh delete mode 100644 scripts/scale_modules_fixed_latents/gemma_llama_xl.sh delete mode 100644 scripts/scale_modules_fixed_latents/gemma_llama_xl_down_ln.sh delete mode 100644 scripts/scale_modules_fixed_latents/gemma_llama_xl_ln_only.sh delete mode 100644 scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down.sh delete mode 100644 scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down_ln.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_2.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_llama.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_r_16.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16_ln.sh delete mode 100644 scripts/scale_modules_no_lightweight/gemma_llama_xl_up_down.sh delete mode 100644 scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl.sh delete mode 100644 scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_down_ln.sh delete mode 100644 scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_ln_only.sh delete mode 100644 scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down.sh delete mode 100644 scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down_ln.sh rename scripts/{fw_qa_3_mini => self_gen_fw_qa_3_mini}/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh (100%) rename scripts/{fw_qa_3_mini => self_gen_fw_qa_3_mini}/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh (100%) diff --git a/chat_templates/google/gemma-2-2b-it.jinja b/chat_templates/google/gemma-2-2b-it.jinja index b9ba380..c29c4e2 100644 --- a/chat_templates/google/gemma-2-2b-it.jinja +++ b/chat_templates/google/gemma-2-2b-it.jinja @@ -13,6 +13,8 @@ {% endif %} {%- if message['role'] == 'user' and loop.first and system_message is defined %} {{ '' + role + '\n' + system_message + '\n\n' + message['content'] | trim + '\n' }} + {%- elif message['role'] == 'assistant' %} + {{ '' + role + '\n' }}{% generation %}{{ message['content'] | trim }}{% endgeneration %}{{ '\n' }} {%- else %} {{ '' + role + '\n' + message['content'] | trim + '\n' }} {%- endif %} diff --git a/chat_templates/meta-llama/Llama-3.1-8B-Instruct.jinja b/chat_templates/meta-llama/Llama-3.1-8B-Instruct.jinja deleted file mode 100644 index 6a5d6c6..0000000 --- a/chat_templates/meta-llama/Llama-3.1-8B-Instruct.jinja +++ /dev/null @@ -1,103 +0,0 @@ -{{- bos_token }} -{%- if custom_tools is defined %} - {%- set tools = custom_tools %} -{%- endif %} -{%- if not tools_in_user_message is defined %} - {%- set tools_in_user_message = true %} -{%- endif %} -{%- if not tools is defined %} - {%- set tools = none %} -{%- endif %} - -{#- This block extracts the system message, so we can slot it into the right place. #} -{%- if messages[0]['role'] == 'system' %} - {%- set system_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} -{%- else %} - {%- set system_message = "" %} -{%- endif %} -{#- System message + builtin tools #} -{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} -{%- if builtin_tools is defined or tools is not none %} - {{- "Environment: ipython\n" }} -{%- endif %} -{%- if builtin_tools is defined %} - {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}} -{%- endif %} -{%- if tools is not none and not tools_in_user_message %} - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} -{%- endif %} -{{- system_message }} -{{- "<|eot_id|>" }} - -{#- Custom tools are passed in a user message with some extra guidance #} -{%- if tools_in_user_message and not tools is none %} - {#- Extract the first user message so we can plug it in here #} - {%- if messages | length != 0 %} - {%- set first_user_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- else %} - {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} -{%- endif %} - {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} - {{- "Given the following functions, please respond with a JSON for a function call " }} - {{- "with its proper arguments that best answers the given prompt.\n\n" }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {{- first_user_message + "<|eot_id|>"}} -{%- endif %} - -{%- for message in messages %} - {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} - {%- elif 'tool_calls' in message %} - {%- if not message.tool_calls|length == 1 %} - {{- raise_exception("This model only supports single tool-calls at once!") }} - {%- endif %} - {%- set tool_call = message.tool_calls[0].function %} - {%- if builtin_tools is defined and tool_call.name in builtin_tools %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- "<|python_tag|>" + tool_call.name + ".call(" }} - {%- for arg_name, arg_val in tool_call.arguments | items %} - {{- arg_name + '="' + arg_val + '"' }} - {%- if not loop.last %} - {{- ", " }} - {%- endif %} - {%- endfor %} - {{- ")" }} - {%- else %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- '{"name": "' + tool_call.name + '", ' }} - {{- '"parameters": ' }} - {{- tool_call.arguments | tojson }} - {{- "}" }} - {%- endif %} - {%- if builtin_tools is defined %} - {#- This means we're in ipython mode #} - {{- "<|eom_id|>" }} - {%- else %} - {{- "<|eot_id|>" }} - {%- endif %} - {%- elif message.role == "tool" or message.role == "ipython" %} - {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} - {%- if message.content is mapping or message.content is iterable %} - {{- message.content | tojson }} - {%- else %} - {{- message.content }} - {%- endif %} - {{- "<|eot_id|>" }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} -{%- endif %} \ No newline at end of file diff --git a/chat_templates/meta-llama/Llama-3.2-11B-Vision-Instruct.jinja b/chat_templates/meta-llama/Llama-3.2-11B-Vision-Instruct.jinja deleted file mode 100644 index 2746432..0000000 --- a/chat_templates/meta-llama/Llama-3.2-11B-Vision-Instruct.jinja +++ /dev/null @@ -1,110 +0,0 @@ -{{- bos_token }} -{%- if custom_tools is defined %} - {%- set tools = custom_tools %} -{%- endif %} -{%- if not tools_in_user_message is defined %} - {%- set tools_in_user_message = true %} -{%- endif %} -{%- if not tools is defined %} - {%- set tools = none %} -{%- endif %} - -{#- This block extracts the system message, so we can slot it into the right place. #} -{%- if messages[0]['role'] == 'system' %} - {%- set system_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- set user_supplied_system_message = true %} -{%- else %} - {%- set system_message = "" %} - {%- set user_supplied_system_message = false %} -{%- endif %} - -{#- Find out if there are any images #} -{% set image_ns = namespace(has_images=false) %} -{%- for message in messages %} - {%- for content in message['content'] %} - {%- if content['type'] == 'image' %} - {%- set image_ns.has_images = true %} - {%- endif %} - {%- endfor %} -{%- endfor %} - -{#- System message if there are no images, or if the user supplied one #} -{%- if user_supplied_system_message or not image_ns.has_images %} - {{- "<|start_header_id|>system<|end_header_id|>\n\n" }} - {%- if tools is not none %} - {{- "Environment: ipython\n" }} - {%- endif %} - {%- if tools is not none and not tools_in_user_message %} - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {%- endif %} - {{- system_message }} - {{- "<|eot_id|>" }} -{%- endif %} - -{#- Custom tools are passed in a user message with some extra guidance #} -{%- if tools_in_user_message and not tools is none %} - {#- Extract the first user message so we can plug it in here #} - {%- if messages | length != 0 %} - {%- set first_user_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- else %} - {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} -{%- endif %} - {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} - {{- "Given the following functions, please respond with a JSON for a function call " }} - {{- "with its proper arguments that best answers the given prompt.\n\n" }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {{- first_user_message + "<|eot_id|>"}} -{%- endif %} - -{%- for message in messages %} - {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }} - {%- if message['content'] is string %} - {{- message['content'] }} - {%- else %} - {%- for content in message['content'] %} - {%- if content['type'] == 'image' %} - {{- '<|image|>' }} - {%- elif content['type'] == 'text' %} - {{- content['text'] }} - {%- endif %} - {%- endfor %} - {%- endif %} - {{- '<|eot_id|>' }} - {%- elif 'tool_calls' in message %} - {%- if not message.tool_calls|length == 1 %} - {{- raise_exception("This model only supports single tool-calls at once!") }} - {%- endif %} - {%- set tool_call = message.tool_calls[0].function %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- '{"name": "' + tool_call.name + '", ' }} - {{- '"parameters": ' }} - {{- tool_call.arguments | tojson }} - {{- "}" }} - {{- "<|eot_id|>" }} - {%- elif message.role == "tool" or message.role == "ipython" %} - {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} - {%- if message.content is mapping or message.content is iterable %} - {{- message.content | tojson }} - {%- else %} - {{- message.content }} - {%- endif %} - {{- "<|eot_id|>" }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} -{%- endif %} \ No newline at end of file diff --git a/chat_templates/meta-llama/Llama-3.2-1B-Instruct.jinja b/chat_templates/meta-llama/Llama-3.2-1B-Instruct.jinja deleted file mode 100644 index c5e7aed..0000000 --- a/chat_templates/meta-llama/Llama-3.2-1B-Instruct.jinja +++ /dev/null @@ -1,83 +0,0 @@ -{{- bos_token }} -{%- if custom_tools is defined %} - {%- set tools = custom_tools %} -{%- endif %} -{%- if not tools_in_user_message is defined %} - {%- set tools_in_user_message = true %} -{%- endif %} -{%- if not tools is defined %} - {%- set tools = none %} -{%- endif %} - -{#- This block extracts the system message, so we can slot it into the right place. #} -{%- if messages[0]['role'] == 'system' %} - {%- set system_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} -{%- else %} - {%- set system_message = "" %} -{%- endif %} -{#- System message #} -{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} -{%- if tools is not none %} - {{- "Environment: ipython\n" }} -{%- endif %} -{%- if tools is not none and not tools_in_user_message %} - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} -{%- endif %} -{{- system_message }} -{{- "<|eot_id|>" }} - -{#- Custom tools are passed in a user message with some extra guidance #} -{%- if tools_in_user_message and not tools is none %} - {#- Extract the first user message so we can plug it in here #} - {%- if messages | length != 0 %} - {%- set first_user_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- else %} - {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} -{%- endif %} - {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} - {{- "Given the following functions, please respond with a JSON for a function call " }} - {{- "with its proper arguments that best answers the given prompt.\n\n" }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {{- first_user_message + "<|eot_id|>"}} -{%- endif %} - -{%- for message in messages %} - {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} - {%- elif 'tool_calls' in message %} - {%- if not message.tool_calls|length == 1 %} - {{- raise_exception("This model only supports single tool-calls at once!") }} - {%- endif %} - {%- set tool_call = message.tool_calls[0].function %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- '{"name": "' + tool_call.name + '", ' }} - {{- '"parameters": ' }} - {{- tool_call.arguments | tojson }} - {{- "}" }} - {{- "<|eot_id|>" }} - {%- elif message.role == "tool" or message.role == "ipython" %} - {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} - {%- if message.content is mapping or message.content is iterable %} - {{- message.content | tojson }} - {%- else %} - {{- message.content }} - {%- endif %} - {{- "<|eot_id|>" }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} -{%- endif %} \ No newline at end of file diff --git a/chat_templates/meta-llama/Llama-3.2-3B-Instruct.jinja b/chat_templates/meta-llama/Llama-3.2-3B-Instruct.jinja deleted file mode 100644 index c5e7aed..0000000 --- a/chat_templates/meta-llama/Llama-3.2-3B-Instruct.jinja +++ /dev/null @@ -1,83 +0,0 @@ -{{- bos_token }} -{%- if custom_tools is defined %} - {%- set tools = custom_tools %} -{%- endif %} -{%- if not tools_in_user_message is defined %} - {%- set tools_in_user_message = true %} -{%- endif %} -{%- if not tools is defined %} - {%- set tools = none %} -{%- endif %} - -{#- This block extracts the system message, so we can slot it into the right place. #} -{%- if messages[0]['role'] == 'system' %} - {%- set system_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} -{%- else %} - {%- set system_message = "" %} -{%- endif %} -{#- System message #} -{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} -{%- if tools is not none %} - {{- "Environment: ipython\n" }} -{%- endif %} -{%- if tools is not none and not tools_in_user_message %} - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} -{%- endif %} -{{- system_message }} -{{- "<|eot_id|>" }} - -{#- Custom tools are passed in a user message with some extra guidance #} -{%- if tools_in_user_message and not tools is none %} - {#- Extract the first user message so we can plug it in here #} - {%- if messages | length != 0 %} - {%- set first_user_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- else %} - {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} -{%- endif %} - {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} - {{- "Given the following functions, please respond with a JSON for a function call " }} - {{- "with its proper arguments that best answers the given prompt.\n\n" }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {{- first_user_message + "<|eot_id|>"}} -{%- endif %} - -{%- for message in messages %} - {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} - {%- elif 'tool_calls' in message %} - {%- if not message.tool_calls|length == 1 %} - {{- raise_exception("This model only supports single tool-calls at once!") }} - {%- endif %} - {%- set tool_call = message.tool_calls[0].function %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- '{"name": "' + tool_call.name + '", ' }} - {{- '"parameters": ' }} - {{- tool_call.arguments | tojson }} - {{- "}" }} - {{- "<|eot_id|>" }} - {%- elif message.role == "tool" or message.role == "ipython" %} - {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} - {%- if message.content is mapping or message.content is iterable %} - {{- message.content | tojson }} - {%- else %} - {{- message.content }} - {%- endif %} - {{- "<|eot_id|>" }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} -{%- endif %} \ No newline at end of file diff --git a/chat_templates/meta-llama/Llama-3.2-90B-Vision-Instruct.jinja b/chat_templates/meta-llama/Llama-3.2-90B-Vision-Instruct.jinja deleted file mode 100644 index 2746432..0000000 --- a/chat_templates/meta-llama/Llama-3.2-90B-Vision-Instruct.jinja +++ /dev/null @@ -1,110 +0,0 @@ -{{- bos_token }} -{%- if custom_tools is defined %} - {%- set tools = custom_tools %} -{%- endif %} -{%- if not tools_in_user_message is defined %} - {%- set tools_in_user_message = true %} -{%- endif %} -{%- if not tools is defined %} - {%- set tools = none %} -{%- endif %} - -{#- This block extracts the system message, so we can slot it into the right place. #} -{%- if messages[0]['role'] == 'system' %} - {%- set system_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- set user_supplied_system_message = true %} -{%- else %} - {%- set system_message = "" %} - {%- set user_supplied_system_message = false %} -{%- endif %} - -{#- Find out if there are any images #} -{% set image_ns = namespace(has_images=false) %} -{%- for message in messages %} - {%- for content in message['content'] %} - {%- if content['type'] == 'image' %} - {%- set image_ns.has_images = true %} - {%- endif %} - {%- endfor %} -{%- endfor %} - -{#- System message if there are no images, or if the user supplied one #} -{%- if user_supplied_system_message or not image_ns.has_images %} - {{- "<|start_header_id|>system<|end_header_id|>\n\n" }} - {%- if tools is not none %} - {{- "Environment: ipython\n" }} - {%- endif %} - {%- if tools is not none and not tools_in_user_message %} - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {%- endif %} - {{- system_message }} - {{- "<|eot_id|>" }} -{%- endif %} - -{#- Custom tools are passed in a user message with some extra guidance #} -{%- if tools_in_user_message and not tools is none %} - {#- Extract the first user message so we can plug it in here #} - {%- if messages | length != 0 %} - {%- set first_user_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- else %} - {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} -{%- endif %} - {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} - {{- "Given the following functions, please respond with a JSON for a function call " }} - {{- "with its proper arguments that best answers the given prompt.\n\n" }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {{- first_user_message + "<|eot_id|>"}} -{%- endif %} - -{%- for message in messages %} - {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }} - {%- if message['content'] is string %} - {{- message['content'] }} - {%- else %} - {%- for content in message['content'] %} - {%- if content['type'] == 'image' %} - {{- '<|image|>' }} - {%- elif content['type'] == 'text' %} - {{- content['text'] }} - {%- endif %} - {%- endfor %} - {%- endif %} - {{- '<|eot_id|>' }} - {%- elif 'tool_calls' in message %} - {%- if not message.tool_calls|length == 1 %} - {{- raise_exception("This model only supports single tool-calls at once!") }} - {%- endif %} - {%- set tool_call = message.tool_calls[0].function %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- '{"name": "' + tool_call.name + '", ' }} - {{- '"parameters": ' }} - {{- tool_call.arguments | tojson }} - {{- "}" }} - {{- "<|eot_id|>" }} - {%- elif message.role == "tool" or message.role == "ipython" %} - {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} - {%- if message.content is mapping or message.content is iterable %} - {{- message.content | tojson }} - {%- else %} - {{- message.content }} - {%- endif %} - {{- "<|eot_id|>" }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} -{%- endif %} \ No newline at end of file diff --git a/chat_templates/meta-llama/Llama-3.3-70B-Instruct.jinja b/chat_templates/meta-llama/Llama-3.3-70B-Instruct.jinja deleted file mode 100644 index 45916f1..0000000 --- a/chat_templates/meta-llama/Llama-3.3-70B-Instruct.jinja +++ /dev/null @@ -1,104 +0,0 @@ -{{- bos_token }} -{%- if custom_tools is defined %} - {%- set tools = custom_tools %} -{%- endif %} -{%- if not tools_in_user_message is defined %} - {%- set tools_in_user_message = true %} -{%- endif %} -{%- if not tools is defined %} - {%- set tools = none %} -{%- endif %} - -{#- This block extracts the system message, so we can slot it into the right place. #} -{%- if messages[0]['role'] == 'system' %} - {%- set system_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} -{%- else %} - {%- set system_message = "" %} -{%- endif %} - -{#- System message + builtin tools #} -{{- "<|start_header_id|>system<|end_header_id|>\n\n" }} -{%- if builtin_tools is defined or tools is not none %} - {{- "Environment: ipython\n" }} -{%- endif %} -{%- if builtin_tools is defined %} - {{- "Tools: " + builtin_tools | reject('equalto', 'code_interpreter') | join(", ") + "\n\n"}} -{%- endif %} -{%- if tools is not none and not tools_in_user_message %} - {{- "You have access to the following functions. To call a function, please respond with JSON for a function call." }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} -{%- endif %} -{{- system_message }} -{{- "<|eot_id|>" }} - -{#- Custom tools are passed in a user message with some extra guidance #} -{%- if tools_in_user_message and not tools is none %} - {#- Extract the first user message so we can plug it in here #} - {%- if messages | length != 0 %} - {%- set first_user_message = messages[0]['content']|trim %} - {%- set messages = messages[1:] %} - {%- else %} - {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }} -{%- endif %} - {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}} - {{- "Given the following functions, please respond with a JSON for a function call " }} - {{- "with its proper arguments that best answers the given prompt.\n\n" }} - {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}.' }} - {{- "Do not use variables.\n\n" }} - {%- for t in tools %} - {{- t | tojson(indent=4) }} - {{- "\n\n" }} - {%- endfor %} - {{- first_user_message + "<|eot_id|>"}} -{%- endif %} - -{%- for message in messages %} - {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %} - {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }} - {%- elif 'tool_calls' in message %} - {%- if not message.tool_calls|length == 1 %} - {{- raise_exception("This model only supports single tool-calls at once!") }} - {%- endif %} - {%- set tool_call = message.tool_calls[0].function %} - {%- if builtin_tools is defined and tool_call.name in builtin_tools %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- "<|python_tag|>" + tool_call.name + ".call(" }} - {%- for arg_name, arg_val in tool_call.arguments | items %} - {{- arg_name + '="' + arg_val + '"' }} - {%- if not loop.last %} - {{- ", " }} - {%- endif %} - {%- endfor %} - {{- ")" }} - {%- else %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}} - {{- '{"name": "' + tool_call.name + '", ' }} - {{- '"parameters": ' }} - {{- tool_call.arguments | tojson }} - {{- "}" }} - {%- endif %} - {%- if builtin_tools is defined %} - {#- This means we're in ipython mode #} - {{- "<|eom_id|>" }} - {%- else %} - {{- "<|eot_id|>" }} - {%- endif %} - {%- elif message.role == "tool" or message.role == "ipython" %} - {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }} - {%- if message.content is mapping or message.content is iterable %} - {{- message.content | tojson }} - {%- else %} - {{- message.content }} - {%- endif %} - {{- "<|eot_id|>" }} - {%- endif %} -{%- endfor %} -{%- if add_generation_prompt %} - {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }} -{%- endif %} \ No newline at end of file diff --git a/configs/context_numbers_10.yaml b/configs/context_numbers_10.yaml deleted file mode 100644 index c29d1ac..0000000 --- a/configs/context_numbers_10.yaml +++ /dev/null @@ -1,68 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 64 -per_device_eval_batch_size: 128 -max_new_tokens: 64 -gen_per_device_eval_batch_size: 128 -max_val_samples_per_ds: 500 -# optim: schedule_free_adamw -learning_rate: 0.0001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_3 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_5 -- data/raw_datasets/context_numbers_6 -- data/raw_datasets/context_numbers_7 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_9 -- data/raw_datasets/context_numbers_10 - -val_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_3 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_5 -- data/raw_datasets/context_numbers_6 -- data/raw_datasets/context_numbers_7 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_9 -- data/raw_datasets/context_numbers_10 - -test_ds_names: -- data/raw_datasets/context_numbers_11 -- data/raw_datasets/context_numbers_12 -- data/raw_datasets/context_numbers_13 -- data/raw_datasets/context_numbers_14 -- data/raw_datasets/context_numbers_15 diff --git a/configs/context_numbers_128.yaml b/configs/context_numbers_128.yaml deleted file mode 100644 index bf7a88d..0000000 --- a/configs/context_numbers_128.yaml +++ /dev/null @@ -1,65 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_48 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_80 -- data/raw_datasets/context_numbers_96 -- data/raw_datasets/context_numbers_112 -- data/raw_datasets/context_numbers_128 - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_128_only.yaml b/configs/context_numbers_128_only.yaml deleted file mode 100644 index 526bdf4..0000000 --- a/configs/context_numbers_128_only.yaml +++ /dev/null @@ -1,56 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_128_big - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_256.yaml b/configs/context_numbers_256.yaml deleted file mode 100644 index 66ece09..0000000 --- a/configs/context_numbers_256.yaml +++ /dev/null @@ -1,66 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_96 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_160 -- data/raw_datasets/context_numbers_192 -- data/raw_datasets/context_numbers_224 -- data/raw_datasets/context_numbers_256 - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_256_only.yaml b/configs/context_numbers_256_only.yaml deleted file mode 100644 index 68489c6..0000000 --- a/configs/context_numbers_256_only.yaml +++ /dev/null @@ -1,56 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_256_big - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_32.yaml b/configs/context_numbers_32.yaml deleted file mode 100644 index 61771db..0000000 --- a/configs/context_numbers_32.yaml +++ /dev/null @@ -1,63 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_12 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_20 -- data/raw_datasets/context_numbers_24 -- data/raw_datasets/context_numbers_28 -- data/raw_datasets/context_numbers_32 - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_32_only.yaml b/configs/context_numbers_32_only.yaml deleted file mode 100644 index 73e2fd7..0000000 --- a/configs/context_numbers_32_only.yaml +++ /dev/null @@ -1,55 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_32_big - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_64.yaml b/configs/context_numbers_64.yaml deleted file mode 100644 index d4bfc03..0000000 --- a/configs/context_numbers_64.yaml +++ /dev/null @@ -1,64 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_24 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_40 -- data/raw_datasets/context_numbers_48 -- data/raw_datasets/context_numbers_56 -- data/raw_datasets/context_numbers_64 - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_64_only.yaml b/configs/context_numbers_64_only.yaml deleted file mode 100644 index dc76775..0000000 --- a/configs/context_numbers_64_only.yaml +++ /dev/null @@ -1,56 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 20 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_64_big - -val_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 diff --git a/configs/context_numbers_debug.yaml b/configs/context_numbers_debug.yaml deleted file mode 100644 index 64ef35b..0000000 --- a/configs/context_numbers_debug.yaml +++ /dev/null @@ -1,47 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -eval_on_start: True -eval_strategy: "steps" -eval_steps: 500 -save_strategy: "no" -# save_steps: 500 -logging_strategy: "steps" -logging_steps: 100 -use_liger_kernel: true -remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -batch_eval_metrics: true - -per_device_train_batch_size: 128 -per_device_eval_batch_size: 128 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_2 - -val_ds_names: -- data/raw_datasets/context_numbers_2 - -test_ds_names: -- data/raw_datasets/context_numbers_2 diff --git a/configs/context_numbers_easy.yaml b/configs/context_numbers_easy.yaml deleted file mode 100644 index ff2e56c..0000000 --- a/configs/context_numbers_easy.yaml +++ /dev/null @@ -1,56 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -add_negative_prompt: false - -eval_on_start: True -eval_strategy: "steps" -eval_steps: 500 -save_strategy: "no" -# save_steps: 500 -logging_strategy: "steps" -logging_steps: 100 -use_liger_kernel: true -remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -batch_eval_metrics: true - -per_device_train_batch_size: 128 -per_device_eval_batch_size: 128 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_3 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_5 - -val_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_3 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_5 - -test_ds_names: -- data/raw_datasets/context_numbers_2 -- data/raw_datasets/context_numbers_3 -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_5 diff --git a/configs/default.yaml b/configs/default.yaml deleted file mode 100644 index bd37401..0000000 --- a/configs/default.yaml +++ /dev/null @@ -1,35 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -add_repeat_prompt: false -eval_on_start: True -eval_strategy: "steps" -eval_steps: 500 -save_strategy: "no" -# save_steps: 500 -logging_strategy: "steps" -logging_steps: 100 -use_liger_kernel: true -remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -batch_eval_metrics: true - -per_device_train_batch_size: 128 -per_device_eval_batch_size: 128 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj diff --git a/configs/fw_qa.yaml b/configs/fw_qa.yaml deleted file mode 100644 index 9195a67..0000000 --- a/configs/fw_qa.yaml +++ /dev/null @@ -1,44 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 - - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa - -val_ds_names: -- fw_qa diff --git a/configs/fw_qa_ctx_qa.yaml b/configs/fw_qa_ctx_qa.yaml deleted file mode 100644 index 941d1e0..0000000 --- a/configs/fw_qa_ctx_qa.yaml +++ /dev/null @@ -1,50 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 - - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa -- ctx_qa -- pwc -- hotpot_qa - -val_ds_names: -- fw_qa -- ctx_qa -- pwc -- hotpot_qa diff --git a/configs/fw_qa_large_ctx_pwc_hotpot.yaml b/configs/fw_qa_large_ctx_pwc_hotpot.yaml deleted file mode 100644 index 4c601b5..0000000 --- a/configs/fw_qa_large_ctx_pwc_hotpot.yaml +++ /dev/null @@ -1,55 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 - - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa -- fw_qa_large -- ctx_qa -- pwc -- hotpot_qa - -val_ds_names: -- fw_qa -- fw_qa_large -- ctx_qa -- pwc -- hotpot_qa - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pwc_hotpot_qa.yaml b/configs/fw_qa_pretrain_mini.yaml similarity index 83% rename from configs/pwc_hotpot_qa.yaml rename to configs/fw_qa_pretrain_mini.yaml index 26da9d3..0c7a92b 100644 --- a/configs/pwc_hotpot_qa.yaml +++ b/configs/fw_qa_pretrain_mini.yaml @@ -23,29 +23,24 @@ max_val_samples_per_ds: 1000 learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 +neftune_noise_alpha: 5 weight_decay: 0.01 - +# warmup_steps: 100 dataloader_prefetch_factor: 8 dataloader_num_workers: 8 - # LoRA lora_r: 8 -lora_dropout: 0.02 +lora_dropout: 0.0 target_modules: - down_proj - # data train_ds_names: - - pwc - - hotpot_qa + - fw_qa_3_mini_pretrain val_ds_names: - - pwc - - hotpot_qa + - fw_qa_3_pretrain -test_ds_names: - - pwc - - hotpot_qa +load_best_model_at_end: true +metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/fw_qa_large_ctx_pwc_hotpot_squad.yaml b/configs/fw_qa_pretrain_mini_and_pwc.yaml similarity index 75% rename from configs/fw_qa_large_ctx_pwc_hotpot_squad.yaml rename to configs/fw_qa_pretrain_mini_and_pwc.yaml index b4c6e93..c392f03 100644 --- a/configs/fw_qa_large_ctx_pwc_hotpot_squad.yaml +++ b/configs/fw_qa_pretrain_mini_and_pwc.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct +model_name_or_path: google/gemma-2-2b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -21,38 +21,29 @@ per_device_eval_batch_size: 8 max_val_samples_per_ds: 1000 # optim: schedule_free_adamw -learning_rate: 0.00002 +learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" neftune_noise_alpha: 5 weight_decay: 0.01 -# +# warmup_steps: 100 dataloader_prefetch_factor: 8 dataloader_num_workers: 8 # LoRA lora_r: 8 -lora_dropout: 0.05 +lora_dropout: 0.0 target_modules: - down_proj - - up_proj - # data train_ds_names: -- fw_qa -- fw_qa_large -- ctx_qa -- pwc -- hotpot_qa -- squad + - fw_qa_3_mini_pretrain + - self_gen/google/gemma-2-2b-it/pwc val_ds_names: -- fw_qa -- fw_qa_large -- ctx_qa -- pwc -- hotpot_qa -- squad + - fw_qa_3_pretrain + - self_gen/google/gemma-2-2b-it/pwc + - pwc load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss +metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/gsm8k.yaml b/configs/fw_qa_pretrain_small.yaml similarity index 82% rename from configs/gsm8k.yaml rename to configs/fw_qa_pretrain_small.yaml index c25f9b8..4224c8a 100644 --- a/configs/gsm8k.yaml +++ b/configs/fw_qa_pretrain_small.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct +model_name_or_path: google/gemma-2-2b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -21,28 +21,26 @@ per_device_eval_batch_size: 8 max_val_samples_per_ds: 1000 # optim: schedule_free_adamw -learning_rate: 0.00002 +learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" neftune_noise_alpha: 5 weight_decay: 0.01 -# +# warmup_steps: 100 dataloader_prefetch_factor: 8 dataloader_num_workers: 8 # LoRA lora_r: 8 -lora_dropout: 0.05 +lora_dropout: 0.0 target_modules: - down_proj - - up_proj - # data train_ds_names: -- gsm8k + - fw_qa_3_small_pretrain val_ds_names: -- gsm8k + - fw_qa_3_pretrain load_best_model_at_end: true -metric_for_best_model: eval_gsm8k_loss +metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/openmathintx.yaml b/configs/fw_qa_pretrain_small_and_pwc.yaml similarity index 75% rename from configs/openmathintx.yaml rename to configs/fw_qa_pretrain_small_and_pwc.yaml index eeb5253..bc3ea17 100644 --- a/configs/openmathintx.yaml +++ b/configs/fw_qa_pretrain_small_and_pwc.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct +model_name_or_path: google/gemma-2-2b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -21,28 +21,29 @@ per_device_eval_batch_size: 8 max_val_samples_per_ds: 1000 # optim: schedule_free_adamw -learning_rate: 0.00002 +learning_rate: 0.00004 # lr_scheduler_type: "constant_with_warmup" neftune_noise_alpha: 5 weight_decay: 0.01 -# +# warmup_steps: 100 dataloader_prefetch_factor: 8 dataloader_num_workers: 8 # LoRA lora_r: 8 -lora_dropout: 0.05 +lora_dropout: 0.0 target_modules: - down_proj - - up_proj - # data train_ds_names: -- openmathintx-2 + - fw_qa_3_small_pretrain + - self_gen/google/gemma-2-2b-it/pwc val_ds_names: -- gsm8k + - fw_qa_3_pretrain + - self_gen/google/gemma-2-2b-it/pwc + - pwc load_best_model_at_end: true -metric_for_best_model: eval_gsm8k_loss +metric_for_best_model: eval_fw_qa_3_pretrain_loss diff --git a/configs/fw_qa_pwc_hotpot.yaml b/configs/fw_qa_pwc_hotpot.yaml deleted file mode 100644 index 117c223..0000000 --- a/configs/fw_qa_pwc_hotpot.yaml +++ /dev/null @@ -1,51 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 - - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa -- pwc -- hotpot_qa - -val_ds_names: -- fw_qa -- pwc -- hotpot_qa - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/fw_qa_tiny.yaml b/configs/fw_qa_tiny.yaml deleted file mode 100644 index 700b845..0000000 --- a/configs/fw_qa_tiny.yaml +++ /dev/null @@ -1,44 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 - - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa_tiny - -val_ds_names: -- fw_qa_tiny diff --git a/configs/hotpot_qa.yaml b/configs/hotpot_qa.yaml deleted file mode 100644 index 10ff35c..0000000 --- a/configs/hotpot_qa.yaml +++ /dev/null @@ -1,45 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- hotpot_qa - -val_ds_names: -- hotpot_qa - -test_ds_names: -- hotpot_qa diff --git a/configs/math_and_code.yaml b/configs/math_and_code.yaml deleted file mode 100644 index 1a9bcf6..0000000 --- a/configs/math_and_code.yaml +++ /dev/null @@ -1,50 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- openmathintx-2 -- opencoder-edu - -val_ds_names: -- gsm8k -- opencoder-edu - -load_best_model_at_end: true -metric_for_best_model: eval_gsm8k_loss diff --git a/configs/pretrain_all.yaml b/configs/pretrain_all.yaml deleted file mode 100644 index b7926fc..0000000 --- a/configs/pretrain_all.yaml +++ /dev/null @@ -1,62 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa -- fw_qa_large -- ctx_qa -- pwc -- hotpot_qa -- squad -- drop -- narrativeqa -- quoref -- ropes -- synthetic_convqa - -val_ds_names: -- fw_qa_large -- ctx_qa -- pwc -- hotpot_qa -- squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pretrain_all_2.yaml b/configs/pretrain_all_2.yaml deleted file mode 100644 index 9b4ae2a..0000000 --- a/configs/pretrain_all_2.yaml +++ /dev/null @@ -1,61 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa_2 -- ctx_qa -- pwc -- hotpot_qa -- squad -- drop -- narrativeqa -- quoref -- ropes -- synthetic_convqa - -val_ds_names: -- fw_qa_xl -- ctx_qa -- pwc -- hotpot_qa -- squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pretrain_all_xl.yaml b/configs/pretrain_all_xl.yaml deleted file mode 100644 index d027400..0000000 --- a/configs/pretrain_all_xl.yaml +++ /dev/null @@ -1,61 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa_xl -- ctx_qa -- pwc -- hotpot_qa -- squad -- drop -- narrativeqa -- quoref -- ropes -- synthetic_convqa - -val_ds_names: -- fw_qa_xl -- ctx_qa -- pwc -- hotpot_qa -- squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pretrain_all_xl_3_medium.yaml b/configs/pretrain_all_xl_3_medium.yaml deleted file mode 100644 index d8b3c5d..0000000 --- a/configs/pretrain_all_xl_3_medium.yaml +++ /dev/null @@ -1,60 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_medium # ~ 130M? - - ctx_qa # 300k - - pwc # 240k - - hotpot_qa # 90k - - squad # 90k - - drop # 77k - - narrativeqa # 40k - - quoref # 11k - - ropes # 11k - - synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3 - - fw_qa_xl - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pretrain_all_xl_3_mini.yaml b/configs/pretrain_all_xl_3_mini.yaml deleted file mode 100644 index 4ede74e..0000000 --- a/configs/pretrain_all_xl_3_mini.yaml +++ /dev/null @@ -1,60 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00004 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.0 -target_modules: - - down_proj -# data -train_ds_names: - - fw_qa_3_mini # 100k - - ctx_qa # 300k - - pwc # 240k - - hotpot_qa # 90k - - squad # 90k - - drop # 77k - - narrativeqa # 40k - - quoref # 11k - - ropes # 11k - - synthetic_convqa # 40k - -val_ds_names: - - fw_qa_3 - - fw_qa_xl - - ctx_qa - - pwc - - hotpot_qa - - squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pretrain_all_xl_and_sum.yaml b/configs/pretrain_all_xl_and_sum.yaml deleted file mode 100644 index 529db76..0000000 --- a/configs/pretrain_all_xl_and_sum.yaml +++ /dev/null @@ -1,63 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 8 -per_device_eval_batch_size: 8 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw - -learning_rate: 0.00002 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 5 -weight_decay: 0.01 -# -warmup_steps: 100 - -dataloader_prefetch_factor: 8 -dataloader_num_workers: 8 -# LoRA -lora_r: 8 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - -# data -train_ds_names: -- fw_qa_xl -- ctx_qa -- pwc -- hotpot_qa -- squad -- drop -- narrativeqa -- quoref -- ropes -- synthetic_convqa -- booksum -- gov_report - -val_ds_names: -- fw_qa_xl -- ctx_qa -- pwc -- hotpot_qa -- squad - -load_best_model_at_end: true -metric_for_best_model: eval_pwc_loss diff --git a/configs/pwc.yaml b/configs/pwc.yaml deleted file mode 100644 index 7344e33..0000000 --- a/configs/pwc.yaml +++ /dev/null @@ -1,45 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 32 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- pwc - -val_ds_names: -- pwc - -test_ds_names: -- pwc diff --git a/configs/pwc_and_ctx_numbers_256.yaml b/configs/pwc_and_ctx_numbers_256.yaml deleted file mode 100644 index 080ff02..0000000 --- a/configs/pwc_and_ctx_numbers_256.yaml +++ /dev/null @@ -1,73 +0,0 @@ -output_dir: "" # just a placeholder -bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct -label_names: ["labels"] -# eval_on_start: True -# eval_strategy: "steps" -# eval_steps: 500 -# save_strategy: "no" -# # save_steps: 500 -# logging_strategy: "steps" -# logging_steps: 100 -# use_liger_kernel: true -# remove_unused_columns: false - -# needed to avoid OOM by compute the metrics batch by batch -# w/o this the trainer stores logits of all sample in memory... -# batch_eval_metrics: true - -per_device_train_batch_size: 32 -per_device_eval_batch_size: 1 -max_val_samples_per_ds: 1000 -# optim: schedule_free_adamw -learning_rate: 0.00001 -# lr_scheduler_type: "constant_with_warmup" -neftune_noise_alpha: 1 -weight_decay: 0.01 - - -# LoRA -lora_r: 16 -lora_dropout: 0.05 -target_modules: - - down_proj - - up_proj - - gate_proj - -# data -train_ds_names: -- pwc -- data/raw_datasets/context_numbers_4 -- data/raw_datasets/context_numbers_8 -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_80 -- data/raw_datasets/context_numbers_96 -- data/raw_datasets/context_numbers_112 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_144 -- data/raw_datasets/context_numbers_160 -- data/raw_datasets/context_numbers_176 -- data/raw_datasets/context_numbers_192 -- data/raw_datasets/context_numbers_208 -- data/raw_datasets/context_numbers_224 -- data/raw_datasets/context_numbers_240 -- data/raw_datasets/context_numbers_256 - - -val_ds_names: -- pwc -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 - -test_ds_names: -- data/raw_datasets/context_numbers_16 -- data/raw_datasets/context_numbers_32 -- data/raw_datasets/context_numbers_64 -- data/raw_datasets/context_numbers_128 -- data/raw_datasets/context_numbers_256 -- pwc diff --git a/configs/pretrain_all_xl_3.yaml b/configs/self_gen_3_small.yaml similarity index 55% rename from configs/pretrain_all_xl_3.yaml rename to configs/self_gen_3_small.yaml index be40251..122ca92 100644 --- a/configs/pretrain_all_xl_3.yaml +++ b/configs/self_gen_3_small.yaml @@ -1,6 +1,6 @@ output_dir: "" # just a placeholder bf16: true -model_name_or_path: meta-llama/Llama-3.2-1B-Instruct +model_name_or_path: google/gemma-2-2b-it label_names: ["labels"] # eval_on_start: True # eval_strategy: "steps" @@ -37,18 +37,24 @@ target_modules: - down_proj # data train_ds_names: - - fw_qa_3 # ~ 267M - - ctx_qa # 300k - - pwc # 240k - - hotpot_qa # 90k - - squad # 90k - - drop # 77k - - narrativeqa # 40k - - quoref # 11k - - ropes # 11k - - synthetic_convqa # 40k + - self_gen/google/gemma-2-2b-it/fw_qa_3_small # ~20M + - self_gen/google/gemma-2-2b-it/ctx_qa # 300k + - self_gen/google/gemma-2-2b-it/pwc # 240k + - self_gen/google/gemma-2-2b-it/hotpot_qa # 90k + - self_gen/google/gemma-2-2b-it/squad # 90k + - self_gen/google/gemma-2-2b-it/drop # 77k + - self_gen/google/gemma-2-2b-it/narrativeqa # 40k + - self_gen/google/gemma-2-2b-it/quoref # 11k + - self_gen/google/gemma-2-2b-it/ropes # 11k + - self_gen/google/gemma-2-2b-it/synthetic_convqa # 40k val_ds_names: + - self_gen/google/gemma-2-2b-it/fw_qa_3 + - self_gen/google/gemma-2-2b-it/fw_qa_xl + - self_gen/google/gemma-2-2b-it/ctx_qa + - self_gen/google/gemma-2-2b-it/pwc + - self_gen/google/gemma-2-2b-it/hotpot_qa + - self_gen/google/gemma-2-2b-it/squad - fw_qa_3 - fw_qa_xl - ctx_qa diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2.sh deleted file mode 100644 index 93df9d7..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --exclude=slurm0-a3nodeset-2 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29560 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_neft1.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_neft1.sh deleted file mode 100644 index 653d6ff..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_neft1.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29560 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=1 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing.sh deleted file mode 100644 index 935c406..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --exclude=slurm0-a3nodeset-2 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_4e-5_lr.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_4e-5_lr.sh deleted file mode 100644 index fa2678b..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_4e-5_lr.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --exclude=slurm0-a3nodeset-2 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_decoder_2.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_decoder_2.sh deleted file mode 100644 index 685fd4d..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_decoder_2.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --exclude=slurm0-a3nodeset-2 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---decoder_depth=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_r16.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_r16.sh deleted file mode 100644 index f039792..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac2_per_layer_processing_r16.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --exclude=slurm0-a3nodeset-2 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4.sh deleted file mode 100644 index 9db5102..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=4 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing.sh deleted file mode 100644 index 9252e4b..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=4 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True --per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_decoder_2.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_decoder_2.sh deleted file mode 100644 index db57cf1..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_decoder_2.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29561 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=4 \ ---decoder_depth=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True --per_layer_processing=True diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_no_reg.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_no_reg.sh deleted file mode 100644 index e93bda7..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_no_reg.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=4 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True --per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.0 diff --git a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_token_mixing.sh b/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_token_mixing.sh deleted file mode 100644 index d68d68d..0000000 --- a/scripts/ablate_arch/gemma_xl_per_rank_gen_fac4_per_layer_processing_token_mixing.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=4 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True --per_layer_processing=True \ ---use_token_mixing=True diff --git a/scripts/fw_qa_3/gemma_per_rank_fac2_per_layer.sh b/scripts/fw_qa_3/gemma_per_rank_fac2_per_layer.sh deleted file mode 100644 index a824d2a..0000000 --- a/scripts/fw_qa_3/gemma_per_rank_fac2_per_layer.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --exclude=slurm0-a3nodeset-2 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl_3.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True diff --git a/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer.sh b/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer.sh deleted file mode 100644 index 08b2e04..0000000 --- a/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=ctxlora_medium -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=8 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=8 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29568 intx_sft.py configs/pretrain_all_xl_3_medium.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_self_attends_per_block=8 --num_latent_factor=2 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True diff --git a/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus.sh b/scripts/fw_qa_pretrain/gemma_pretrain.sh similarity index 56% rename from scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus.sh rename to scripts/fw_qa_pretrain/gemma_pretrain.sh index a26d24a..1ce7fb6 100644 --- a/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus.sh +++ b/scripts/fw_qa_pretrain/gemma_pretrain.sh @@ -14,20 +14,22 @@ # export OMP_NUM_THREADS=24 # export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora +# . ~/miniconda3/etc/profile.d/conda.sh +# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora # eval "$@" -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/pretrain_all_xl_3_medium.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=32 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29560 intx_sft.py configs/fw_qa_pretrain_small.yaml \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=2 --per_device_train_batch_size=4 \ +--gradient_accumulation_steps=32 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \ --target_modules=down_proj \ --num_self_attends_per_block=8 --num_latent_factor=2 \ --lora_r=8 \ --eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ --neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ +--add_negative_prompt=False \ --add_repeat_prompt=False \ --use_sequence_packing=True --per_rank_gen=True \ ---per_layer_processing=True +--per_layer_processing=True \ +--gen_lora_l1_reg_coef=0.1 \ + diff --git a/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh b/scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh similarity index 62% rename from scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh rename to scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh index 5193bf3..067937a 100644 --- a/scripts/fw_qa_3/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh +++ b/scripts/fw_qa_pretrain/gemma_pretrain_and_pwc.sh @@ -14,21 +14,22 @@ # export OMP_NUM_THREADS=24 # export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora +# . ~/miniconda3/etc/profile.d/conda.sh +# conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora # eval "$@" -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/pretrain_all_xl_3_medium.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ +uv run accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ +--gpu_ids all --main_process_port 29560 intx_sft.py configs/fw_qa_pretrain_small_and_pwc.yaml \ +--model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1 --per_device_train_batch_size=4 \ +--gradient_accumulation_steps=16 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver \ --target_modules=down_proj \ --num_self_attends_per_block=8 --num_latent_factor=2 \ --lora_r=8 \ --eval_steps=5000 --save_steps=5000 --learning_rate=4e-5 --lora_dropout=0.0 \ --neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ +--add_negative_prompt=False \ --add_repeat_prompt=False \ --use_sequence_packing=True --per_rank_gen=True \ --per_layer_processing=True \ ---gen_lora_l1_reg_coef=0.1 +--gen_lora_l1_reg_coef=0.1 \ + diff --git a/scripts/repeat_text/gemma_llama_xl.sh b/scripts/repeat_text/gemma_llama_xl.sh deleted file mode 100644 index cb52901..0000000 --- a/scripts/repeat_text/gemma_llama_xl.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=True \ ---use_sequence_packing=True --ctx_encoder_model_name_or_path=meta-llama/Llama-3.2-3B-Instruct diff --git a/scripts/scale_modules_fixed_latents/gemma_llama_xl.sh b/scripts/scale_modules_fixed_latents/gemma_llama_xl.sh deleted file mode 100644 index cb6c811..0000000 --- a/scripts/scale_modules_fixed_latents/gemma_llama_xl.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29561 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_fixed_latents/gemma_llama_xl_down_ln.sh b/scripts/scale_modules_fixed_latents/gemma_llama_xl_down_ln.sh deleted file mode 100644 index a75e91c..0000000 --- a/scripts/scale_modules_fixed_latents/gemma_llama_xl_down_ln.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29566 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=2 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_fixed_latents/gemma_llama_xl_ln_only.sh b/scripts/scale_modules_fixed_latents/gemma_llama_xl_ln_only.sh deleted file mode 100644 index 8506a41..0000000 --- a/scripts/scale_modules_fixed_latents/gemma_llama_xl_ln_only.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=None --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=4 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down.sh b/scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down.sh deleted file mode 100644 index b78380c..0000000 --- a/scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj,up_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=4 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down_ln.sh b/scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down_ln.sh deleted file mode 100644 index cf90799..0000000 --- a/scripts/scale_modules_fixed_latents/gemma_llama_xl_up_down_ln.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=10.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj,up_proj --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=2 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_2.sh b/scripts/scale_modules_no_lightweight/gemma_llama_2.sh deleted file mode 100644 index 3b5a04b..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_2.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_2.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=1.0 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --ctx_encoder_model_name_or_path=meta-llama/Llama-3.2-3B-Instruct diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl.sh deleted file mode 100644 index bb55381..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --ctx_encoder_model_name_or_path=meta-llama/Llama-3.2-3B-Instruct diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen.sh deleted file mode 100644 index 4605e1c..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=16 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1.sh deleted file mode 100644 index 70de581..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=4 \ ---lora_r=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_llama.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_llama.sh deleted file mode 100644 index 7dbb986..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_llama.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=2 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True \ ---ctx_encoder_model_name_or_path=meta-llama/Llama-3.2-3B-Instruct - diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_r_16.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_r_16.sh deleted file mode 100644 index 4421159..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_per_rank_gen_latent_fac_1_r_16.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=8 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=8 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=1 \ ---lora_r=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True --per_rank_gen=True diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16.sh deleted file mode 100644 index 2e8ae06..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/pretrain_all_xl_and_sum.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_r=16 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16_ln.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16_ln.sh deleted file mode 100644 index fda30d8..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_r_16_ln.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl_and_sum.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_r=16 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_no_lightweight/gemma_llama_xl_up_down.sh b/scripts/scale_modules_no_lightweight/gemma_llama_xl_up_down.sh deleted file mode 100644 index fa3ce26..0000000 --- a/scripts/scale_modules_no_lightweight/gemma_llama_xl_up_down.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=16 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl_and_sum.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=16 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=up_proj,down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_r=8 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl.sh b/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl.sh deleted file mode 100644 index 6614d86..0000000 --- a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29571 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=16 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=3e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=False \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_down_ln.sh b/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_down_ln.sh deleted file mode 100644 index 265f185..0000000 --- a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_down_ln.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29565 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_ln_only.sh b/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_ln_only.sh deleted file mode 100644 index 785207c..0000000 --- a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_ln_only.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29564 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=None --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down.sh b/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down.sh deleted file mode 100644 index dbe9e5e..0000000 --- a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29562 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj,up_proj \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down_ln.sh b/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down_ln.sh deleted file mode 100644 index ba763f6..0000000 --- a/scripts/scale_modules_scale_latents_no_lightweight/gemma_llama_xl_up_down_ln.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/bin/bash -#SBATCH --job-name=gemma_llama_instruct -#SBATCH --partition=a3 -#SBATCH --nodes=1 -#SBATCH --gpus=4 -#SBATCH --output=outputs/%x-%j.out -#SBATCH --error=outputs/%x-%j.out - -# module load -# module load cuda/12.1 -# module load cudnn/8.9.7 -# module load nccl/cuda-12.1/2.18.3 -# module load hpcx/2.20 - -# export OMP_NUM_THREADS=24 -# export TRITON_CACHE_DIR=/tmp/.triton/ -. ~/miniconda3/etc/profile.d/conda.sh -conda activate /home/rujikorn_sakana_ai/.conda/envs/ctx-to-lora -# eval "$@" - -accelerate launch --num_processes=4 --gradient_accumulation_steps=4 --gradient_clipping=1.0 \ ---gpu_ids all --main_process_port 29563 intx_sft.py configs/pretrain_all_xl.yaml \ ---model_name_or_path=google/gemma-2-2b-it --num_train_epochs=5.1 --per_device_train_batch_size=32 \ ---gradient_accumulation_steps=4 --per_device_eval_batch_size=32 --exp_setup=hyper_lora --aggregator_type=perceiver \ ---target_modules=down_proj,up_proj --extra_modules=input_layernorm,post_attention_layernorm \ ---num_blocks=1 --num_self_attends_per_block=8 --num_latent_factor=8 \ ---eval_steps=5000 --save_steps=5000 --learning_rate=2e-5 --lora_dropout=0.0 \ ---neftune_noise_alpha=5 --use_light_weight_lora=True --light_weight_latent_size=512 \ ---load_best_model_at_end=True --metric_for_best_model=pwc_loss --add_negative_prompt=False \ ---add_repeat_prompt=False \ ---use_sequence_packing=True diff --git a/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh b/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh similarity index 100% rename from scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh rename to scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg.sh diff --git a/scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh b/scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh similarity index 100% rename from scripts/fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh rename to scripts/self_gen_fw_qa_3_mini/medium_gemma_per_rank_fac2_per_layer_4_gpus_gen_lora_reg_repeat.sh