versioning + cli commands

.gitignore

@@ -5,13 +5,12 @@ openai_batches/
 models/
 results/
 datasets/
-data/
+/data/
 __pycache__/
 *egg-info/
 .vscode/
 .ipynb_checkpoints/
 wandb/
-data/
 *.bak
 *outputs/
 plots/

README.md

@@ -27,6 +27,7 @@ input_ids = model.tokenize(query)
 outputs = model.generate(ctx_ids, input_ids)
 print(model.decode(outputs))
 ```
+
 ## 🏋️ Training
 ### 🔢 HyperLoRA w/ context_numbers_10
 ```bash
@@ -37,15 +38,41 @@ WANDB_MODE=disabled uv run python intx_sft.py configs/context_numbers_10.yaml --
 ```bash
 WANDB_MODE=disabled run python hyperlora/intx_sft.py configs/context_numbers_128.yaml --model_name_or_path=meta-llama/Llama-3.2-1B-Instruct --num_train_epochs=10 --per_device_train_batch_size=64 --per_device_eval_batch_size=8 --exp_setup=hyper_lora --aggregator_type=perceiver --target_modules=down_proj
 ``` -->
+### Favourite numbers data generation
+```bash
+uv run python data/generate_fav_num.py
+# or
+uv run python data/generate_fav_num_big.py
+```
 
-### Generate fineweb qa
+
+### Data Generation
 ```bash
 # this might take several days...
 # python process_fineweb.py
 # python generate_fw_qa.py
 # python post_process_fw_qa.py
 conda activate vllm;
-vllm_model=mistralai/Mistral-Small-3.1-24B-Instruct-2503 run python generate_fw_edu_qa_vllm.py "0*_*" 3
+# [WIP] download fineweb-edu
+vllm_model=mistralai/Mistral-Small-3.1-24B-Instruct-2503 uv run python generate_fw_edu_qa_vllm.py "0*_*" 3
+# [WIP] there are other datasets where we used openAI to gen QA pairs
+
+# self-generate SFT data
+# gemma-2-2b-it
+vllm_model=google/gemma-2-2b-it uv run python data/self_generate_qa.py "0*_*"
+# each context has 3 chat instances as training data
+# the responses are not the same as the ones in the pretraining though
+
+
+# pre-training data
+# better to include QAs at the end of pretrain docs
+# # https://arxiv.org/pdf/2406.14491
+uv run python data/generate_pretrain_from_fw_qa.py
+# we could do this online but
+# will have duplicate contexts (each context is used to generate 3 QA pairs)
+# better if we do this offline bc dedup the contexts
+
+
 ```
 
 <!-- 

install.sh

@@ -8,11 +8,10 @@
 # # git checkout f724be699e8adf7ca8004ea0e519dfac83a06f18
 # # pip install -e .
 uv venv
-uv pip install torch==2.6.0 torchvision==0.21.0 torchaudio==2.6.0 --index-url https://download.pytorch.org/whl/cu126
+uv pip install torch==2.5.1 torchvision==0.20.1 torchaudio==2.5.1 --index-url https://download.pytorch.org/whl/cu124
 uv sync
-# wget https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.3/flash_attn-2.7.3+cu12torch2.6cxx11abiTRUE-cp310-cp310-linux_x86_64.whl
-uv pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.3/flash_attn-2.7.3+cu12torch2.6cxx11abiTRUE-cp310-cp310-linux_x86_64.whl
+uv pip install https://github.com/Dao-AILab/flash-attention/releases/download/v2.7.4.post1/flash_attn-2.7.4.post1+cu12torch2.5cxx11abiTrue-cp310-cp310-linux_x86_64.whl
 # uv pip install flash-attn==2.6.3 --no-build-isolation
 
 # dev
-pre-commit install
+uv run pre-commit install

pyproject.toml

@@ -52,7 +52,8 @@ exclude = [
     "LongBench",
     "scripts",
     "train_outputs",
-    "data",
+    "./data/",
+    "/data/",
     "generated_tasks",
     "outputs",
     "plots",
@@ -68,3 +69,6 @@ ignore = ["E", "F"]
 
 [tool.isort]
 profile = "black"
+known_first_party = ["wandb"]
+known_local_folder = ["ctx_to_lora"]
+known_third_party = ["wandb"]