mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-05-30 14:35:15 +02:00
11 lines
284 B
Python
11 lines
284 B
Python
|
from huggingface_hub import snapshot_download
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
fw_dir = "./data/raw_datasets/fineweb_edu/"
|
||
|
|
snapshot_download(
|
||
|
|
"HuggingFaceFW/fineweb-edu",
|
||
|
|
repo_type="dataset",
|
||
|
|
local_dir=fw_dir,
|
||
|
|
allow_patterns="sample/100BT/*",
|
||
|
|
)
|