apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

more trainer args + fix columns check

Browse source
This commit is contained in:
51616 2024-12-21 15:24:02 +00:00
parent 33aefa2a0c
commit c87c10aaf1
2 changed files with 9 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
configs/default.yaml
View file

@ -13,8 +13,12 @@ use_liger_kernel: true
remove_unused_columns: false
optim: schedule_free_adamw
learning_rate: 0.00001
learning_rate: 0.0001
neftune_noise_alpha: 5
label_smoothing_factor: 0.1
weight_decay: 0.01
warmup_ratio: 0.1
# LoRA

7
hyperlora/intx_sft.py
View file

@ -204,9 +204,10 @@ def main():
def validate_columns(tokenized_ds):
ref_cols = set(
["input_ids", "attention_mask", "labels", "ctx_features", "ctx_attn_mask"]
)
cols = ["input_ids", "attention_mask", "labels"]
if "ctx_features" in tokenized_ds["train"].column_names:
cols += ["ctx_features", "ctx_attn_mask"]
ref_cols = set(cols)
assert (
set(tokenized_ds["train"].column_names) == ref_cols
), f"Columns mismatch: {set(tokenized_ds['train'].column_names)} != {ref_cols}"