mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
d99fa8732b
- work with vllm self-gen data (everything is tokenized during self-gen) - fix chat template! - not use liger kernel for distillation (for both training and eval)
9 lines
244 B
Python
9 lines
244 B
Python
from huggingface_hub import snapshot_download
|
|
|
|
if __name__ == "__main__":
|
|
fw_dir = "./data/raw_datasets/self_gen/"
|
|
snapshot_download(
|
|
"Rujikorn/self_gen_qa_logprobs",
|
|
repo_type="dataset",
|
|
local_dir=fw_dir,
|
|
)
|