diff --git a/README.md b/README.md index 51a7586..cb40411 100644 --- a/README.md +++ b/README.md @@ -1 +1,6 @@ -[Project doc](https://docs.google.com/document/d/1RCQDzlVU7YGoTwR84gLQfhxTv0RFnW6srQlqfCC2bvQ/edit?usp=sharing) \ No newline at end of file +[Project doc](https://docs.google.com/document/d/1RCQDzlVU7YGoTwR84gLQfhxTv0RFnW6srQlqfCC2bvQ/edit?usp=sharing) + +### Finetuning the base model with LoRA adaptor +```bash +python hyperlora/intx_sft.py configs/default.yaml --model_name_or_path=meta-llama/Llama-3.2-1B-Instruct --num_train_epochs=100 --per_device_train_batch_size=8 --per_device_eval_batch_size=32 --overwrite_output_dir=True +``` diff --git a/hyperlora/data_utils.py b/hyperlora/data_utils.py index 6dd84a7..7c81b47 100644 --- a/hyperlora/data_utils.py +++ b/hyperlora/data_utils.py @@ -314,7 +314,7 @@ def tokenize_chat_messages( text, return_offsets_mapping=mask_assistant_inputs, add_special_tokens=False, - **tokenizer_kwargs if tokenizer_kwargs is not None else {}, + **(tokenizer_kwargs or {}), ) if mask_assistant_inputs: assistant_ranges = get_assistant_start_end_indices(messages, text) diff --git a/hyperlora/model_loading.py b/hyperlora/model_loading.py index b15249b..8835211 100644 --- a/hyperlora/model_loading.py +++ b/hyperlora/model_loading.py @@ -101,7 +101,6 @@ def get_model( device="cuda:0", dtype=torch.bfloat16, ): - # TODO: we could use lower precision during finetuning model_init_kwargs = dict( pretrained_model_name_or_path=model_name_or_path, device_map=device,