diff --git a/data/download_generated_fineweb_qa.py b/data/download_generated_fineweb_qa.py new file mode 100644 index 0000000..f5ceb2e --- /dev/null +++ b/data/download_generated_fineweb_qa.py @@ -0,0 +1,9 @@ +from huggingface_hub import snapshot_download + +if __name__ == "__main__": + fw_dir = "./data/raw_datasets/fw_qa_v2/" + snapshot_download( + "SakanaAI/fineweb_qa", + repo_type="dataset", + local_dir=fw_dir, + )