apunkt/doc-to-lora
1
0
Fork 0
mirror of https://github.com/SakanaAI/doc-to-lora.git synced 2026-07-23 17:01:04 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

toy ctx nums and multi-lora training (#11)

Browse source 
* multi-lora trainable toy number repeat dataset

* per rank bias init

* remove head_bias +simplify merge + skip perplexities metric

* ctx_numbers train example

* self-gen ctx numbers example
This commit is contained in:
Rujikorn Charakorn 2025-08-18 18:38:07 +09:00 • committed by GitHub
parent dfee685678
commit c5e9bc769d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
36 changed files with 1172 additions and 231 deletions
Download patch file Download diff file Expand all files Collapse all files

6
train.py
View file

@ -199,6 +199,7 @@ def main():
base_model,
hypernet_config,
ctx_encoder_args,
lora_aggregation=ctx_args.lora_aggregation,
)
else:
@ -249,6 +250,9 @@ def main():
ctx_model_max_len = model.ctx_encoder.config.max_position_embeddings
if ctx_args.max_ctx_len > 0:
ctx_model_max_len = ctx_args.max_ctx_len
if ctx_args.max_ctx_chunk_len <= 0:
# set default chunk size to max length of the ctx encoder
ctx_args.max_ctx_chunk_len = ctx_model_max_len
_get_tokenized_dataset = partial(
get_tokenized_dataset,
@ -260,6 +264,8 @@ def main():
ctx_tokenizer=ctx_tokenizer,
add_ctx_to_chat=add_ctx_to_chat,
max_ctx_chunk_len=ctx_args.max_ctx_chunk_len,
min_ctx_chunk_len=ctx_args.min_ctx_chunk_len,
random_chunking=ctx_args.random_chunking,
max_ctx_chunk_num=ctx_args.max_ctx_chunk_num,
use_kl_loss=ctx_args.use_kl_loss,
)