diff --git a/README.md b/README.md index 5587336..ffae39d 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,7 @@ uv run python data/generate_fav_num_big.py ***Download a subset of generated Fineweb-QA directly*** ```bash uv run data/download_generated_fineweb_qa.py +uv run data/download_self_gen_qa.py ``` ***Generate data from scratch*** diff --git a/data/download_self_gen_qa.py b/data/download_self_gen_qa.py new file mode 100644 index 0000000..68d98e6 --- /dev/null +++ b/data/download_self_gen_qa.py @@ -0,0 +1,9 @@ +from huggingface_hub import snapshot_download + +if __name__ == "__main__": + fw_dir = "./data/raw_datasets/self_gen/" + snapshot_download( + "SakanaAI/self_gen_qa", + repo_type="dataset", + local_dir=fw_dir, + )