apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-26 17:11:02 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor config + schedule free adam + liger kernel

Browse source
This commit is contained in:
51616 2024-12-21 12:39:03 +00:00
parent c312d4bafb
commit 7980420114
6 changed files with 43 additions and 22 deletions
Download patch file Download diff file Expand all files Collapse all files

4
hyperlora/model_loading.py
View file

@ -128,13 +128,13 @@ def get_model(
def get_lora_config(model_dir, **kwargs):
r = kwargs.get("r", 8)
r = kwargs.pop("lora_r", 8)
peft_conf_kwargs = dict(
r=r,
peft_type=PeftType.LORA,
base_model_name_or_path=model_dir,
task_type="CAUSAL_LM",
lora_dropout=0.05,
lora_dropout=kwargs.get("lora_dropout", 0.05),
lora_alpha=r ** (3 / 2) * 2,
)