From 7fa77b1759db22e0da6aca7f7742bbe0adad8f61 Mon Sep 17 00:00:00 2001 From: Sarvesh Date: Tue, 9 Jun 2026 15:37:50 +0530 Subject: [PATCH] Add self-contained Video2LoRA zero-visual-token inference demo notebook --- demo.ipynb | 543 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 543 insertions(+) create mode 100644 demo.ipynb diff --git a/demo.ipynb b/demo.ipynb new file mode 100644 index 0000000..b64a9ab --- /dev/null +++ b/demo.ipynb @@ -0,0 +1,543 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Video2LoRA: Self-Contained Zero-Visual-Token Inference Tutorial\n", + "\n", + "This notebook provides a **completely self-contained, single-file tutorial** for running zero-visual-token inference with **Video2LoRA**. Under the hood, Video2LoRA utilizes a perceiver hypernetwork to read a video's intermediate activation features layer-by-layer and directly predict a set of Low-Rank Adaptation (LoRA) weights in a single forward pass.\n", + "\n", + "Once the adapter weights are generated, they are injected into the base Vision-Language Model (VLM), which can then answer arbitrary text queries **without requiring the original visual tokens** in its context window.\n", + "\n", + "All model architectures, helper functions, and dependencies are defined directly within this notebook so it can run independently of the main repository codebase.\n", + "\n", + "### \ud83d\udd17 Project Resources\n", + "- [**\ud83c\udf10 Project Website**](https://video2lora.github.io/)\n", + "- [**\ud83d\udcc4 arXiv Paper**](https://arxiv.org/abs/2606.04351)\n", + "- [**\ud83e\udd17 Hugging Face Checkpoints**](https://huggingface.co/MananSuri27/Video2LoRA-SmolVLM-ckpts)\n", + "- [**\ud83d\udcbb GitHub Repository**](https://github.com/MananSuri27/video2lora)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 0: Package Installation\n", + "\n", + "First, we install all the required open-source packages in the environment." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install torch torchvision torchaudio\n", + "!pip install transformers accelerate peft huggingface_hub decord av opencv-python matplotlib einops jaxtyping" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Model Architectures & Utilities\n", + "\n", + "Here, we define the full Video2LoRA model architecture classes (including the perceiver aggregator, hypernet heads, model load patches, and custom LoRA injection layers) directly inside the notebook. This makes the notebook completely independent of external python code files." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "import os\n", + "from pathlib import Path\n", + "\n", + "# Add repository root and src directory to sys.path to enable local imports\n", + "repo_root = str(Path(os.getcwd()).resolve())\n", + "if repo_root not in sys.path:\n", + " sys.path.insert(0, repo_root)\n", + "src_dir = str(Path(repo_root) / \"src\")\n", + "if os.path.exists(src_dir) and src_dir not in sys.path:\n", + " sys.path.insert(0, src_dir)\n", + "\n", + "import torch\n", + "import cv2\n", + "import matplotlib.pyplot as plt\n", + "from dataclasses import dataclass\n", + "from typing import List, Optional\n", + "\n", + "# Import custom Video2LoRA architectures and utilities from the repository modules\n", + "from ctx_to_lora.modeling.lora_layer import apply_lora_to_layers\n", + "from ctx_to_lora.modeling.lora_merger import combine_lora\n", + "from scripts.video2lora.train_smolvlm_stage1 import build_stage1_model\n", + "from scripts.video2lora.train_smolvlm_online import (\n", + " prepare_smolvlm_inputs,\n", + " extract_l2l_fused_text_features,\n", + ")\n", + "\n", + "@dataclass\n", + "class TrainArgs:\n", + " smolvlm_name_or_path: str\n", + " train_manifest: str\n", + " val_manifest: str\n", + " output_dir: str\n", + " lora_r: int = 16\n", + " lora_dropout: float = 0.0\n", + " target_modules: Optional[List[str]] = None\n", + " latent_size: int = 512\n", + " dropout_rate: float = 0.0\n", + " n_latent_queries: int = 8\n", + " num_blocks: int = 9\n", + " num_self_attn_per_block: int = 0\n", + " video_fps: Optional[float] = None\n", + " max_frames: int = 12\n", + " video_size_longest_edge: int = 384\n", + " video_load_backend: str = \"auto\"\n", + " internalization_prompt: str = \"Internalize this video for later captioning.\"\n", + " kl_weight: float = 0.0\n", + " generation_max_new_tokens: int = 128\n", + "\n", + "print(\"Video2LoRA imported architectures and utility functions successfully.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Load and Inspect Manifest\n", + "\n", + "Video2LoRA uses a standard JSONL manifest file for training and evaluation. Below, we define a sample manifest row matching the child watering plants example from the paper's qualitative evaluation." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Define an example dataset manifest entry\n", + "sample_example = {\n", + " \"id\": \"sample-watering-plants\",\n", + " \"video_path\": \"raw/finevideo/sample_watering_plants.mp4\",\n", + " \"dataset\": \"finevideo\",\n", + " \"task_type\": \"caption\",\n", + " \"prompt\": \"Describe the key visible events in chronological order. Include all important actions and changes you can observe, with enough detail to distinguish each event clearly.\",\n", + " \"target_text\": \"Little boy watering plants outdoors; Using watering can to pour water into flower pot; Shifting camera angle from side view to rear view; Tapping edge of flower pot a few times; Setting down watering can\"\n", + "}\n", + "\n", + "print(f\"Video ID: {sample_example['id']}\")\n", + "print(f\"Prompt: {sample_example['prompt']}\")\n", + "print(f\"Ground Truth: {sample_example['target_text']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Visualize Video Frames\n", + "\n", + "Let's write a utility function to extract and display keyframes from the input video file using OpenCV and Matplotlib." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def show_video_frames(video_path, num_frames=4):\n", + " \"\"\"\n", + " Helper function to load keyframes from a video path and plot them in a grid.\n", + " \"\"\"\n", + " if not os.path.exists(video_path):\n", + " print(f\"Video file not found at: {video_path} (Please provide a valid video to inspect).\")\n", + " return\n", + " \n", + " cap = cv2.VideoCapture(video_path)\n", + " total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n", + " indices = [int(i * total_frames / num_frames) for i in range(num_frames)]\n", + " \n", + " fig, axes = plt.subplots(1, num_frames, figsize=(16, 4))\n", + " for i, idx in enumerate(indices):\n", + " cap.set(cv2.CAP_PROP_POS_FRAMES, idx)\n", + " ret, frame = cap.read()\n", + " if ret:\n", + " frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n", + " axes[i].imshow(frame)\n", + " axes[i].axis('off')\n", + " axes[i].set_title(f\"Frame {idx}\")\n", + " else:\n", + " axes[i].text(0.5, 0.5, \"Frame Read Error\", ha=\"center\", va=\"center\")\n", + " axes[i].axis('off')\n", + " plt.tight_layout()\n", + " plt.show()\n", + " cap.release()\n", + "\n", + "# To run the visualization, uncomment the line below:\n", + "show_video_frames(sample_example[\"video_path\"], num_frames=4)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Download Video2LoRA Checkpoint\n", + "\n", + "We use the `huggingface_hub` client to download the 2.2B Video2LoRA checkpoint weights (`video2lora-smolvlm2-2.2b-best-ce.pt`) directly from Hugging Face." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Downloading 2.2B Video2LoRA Checkpoint from Hugging Face...\")\n", + "\n", + "checkpoint_dir = Path(\"checkpoints/Video2LoRA-SmolVLM-ckpts\")\n", + "checkpoint_dir.mkdir(parents=True, exist_ok=True)\n", + "\n", + "# Download checkpoint weights programmatically\n", + "ckpt_path = hf_hub_download(\n", + " repo_id=\"MananSuri27/Video2LoRA-SmolVLM-ckpts\",\n", + " filename=\"video2lora-smolvlm2-2.2b-best-ce.pt\",\n", + " local_dir=str(checkpoint_dir)\n", + ")\n", + "\n", + "print(f\"Checkpoint successfully downloaded to: {ckpt_path}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: Load Base VLM & Initialize Modulated Model\n", + "\n", + "Configure `TrainArgs` matching the 2.2B preset, load the base SmolVLM2 model, processor, and tokenizer, and load the hypernetwork checkpoint." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Set configuration arguments for the 2.2B SmolVLM2 model preset\n", + "train_args = TrainArgs(\n", + " smolvlm_name_or_path=\"HuggingFaceTB/SmolVLM2-2.2B-Instruct\",\n", + " train_manifest=\"\",\n", + " val_manifest=\"\",\n", + " output_dir=\"\",\n", + " lora_r=16,\n", + " lora_dropout=0.0,\n", + " target_modules=[\"down_proj\"],\n", + " latent_size=512,\n", + " dropout_rate=0.0,\n", + " n_latent_queries=8,\n", + " num_blocks=9,\n", + " num_self_attn_per_block=0,\n", + " video_fps=None,\n", + " max_frames=12,\n", + " video_size_longest_edge=384,\n", + " video_load_backend=\"auto\",\n", + " internalization_prompt=\"Internalize this video for later captioning.\",\n", + " kl_weight=0.0,\n", + " generation_max_new_tokens=128\n", + ")\n", + "\n", + "# Configure device\n", + "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", + "\n", + "# 1. Initialize SmolVLM2 base model, processor, and tokenizer\n", + "print(\"Loading SmolVLM2 model (2.2B-Instruct)...\")\n", + "model, raw_model, processor, tokenizer = build_stage1_model(train_args, device=device)\n", + "\n", + "# 2. Load the hypernetwork weights\n", + "print(\"Loading hypernetwork weights...\")\n", + "state_dict = torch.load(ckpt_path, map_location=\"cpu\", weights_only=False)\n", + "model.load_state_dict(state_dict)\n", + "\n", + "# Place models in evaluation mode\n", + "model.to(device)\n", + "model.eval()\n", + "raw_model.eval()\n", + "\n", + "print(\"Model load complete.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: Parametric Video Internalization\n", + "\n", + "We process the video frames through the VLM visual encoder to extract activations, then generate the custom LoRA adapter weights in a single pass of the hypernetwork." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1. Define internalization messages\n", + "internalize_messages = [\n", + " [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"video\", \"path\": sample_example[\"video_path\"]},\n", + " {\"type\": \"text\", \"text\": train_args.internalization_prompt}\n", + " ]\n", + " }\n", + " ]\n", + "]\n", + "\n", + "# 2. Process video and prompt messages into VLM input tensors\n", + "vlm_inputs = prepare_smolvlm_inputs(\n", + " processor,\n", + " internalize_messages,\n", + " device,\n", + " video_fps=train_args.video_fps,\n", + " max_frames=train_args.max_frames,\n", + " video_size_longest_edge=train_args.video_size_longest_edge,\n", + " video_load_backend=train_args.video_load_backend\n", + ")\n", + "\n", + "# 3. Extract layer-by-layer hidden features from the vision encoder\n", + "ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(\n", + " raw_model,\n", + " vlm_inputs,\n", + " num_target_layers=model.hypernet.n_layers\n", + ")\n", + "\n", + "# 4. Generate custom LoRA adapter weights\n", + "print(\"Generating dynamic LoRA weights...\")\n", + "generated_loras, _ = model.generate_weights(\n", + " ctx_ids=None,\n", + " ctx_features=ctx_features,\n", + " ctx_attn_mask=ctx_attn_mask,\n", + " ctx_position_ids=ctx_position_ids\n", + ")\n", + "\n", + "# Combine generated adapter modules\n", + "generated_loras = combine_lora(\n", + " generated_loras,\n", + " torch.ones(1, dtype=torch.int32, device=device),\n", + " lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None\n", + ")\n", + "\n", + "print(\"LoRA adapter weights generated successfully.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7: Run QA Inference (Base Model vs Video2LoRA)\n", + "\n", + "To perform a fair comparison, we first run inference with the **Base Model** (which requires processing the full visual video tokens in context). Afterwards, we run inference with **Video2LoRA** (which bypasses visual tokens completely and queries the model using only text, utilizing the dynamic LoRA adapter)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# === Part A: Base Model Inference (with visual tokens) ===\n", + "\n", + "# 1. Format prompt for the base VLM (requires video inputs)\n", + "base_messages = [\n", + " [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"video\", \"path\": sample_example[\"video_path\"]},\n", + " {\"type\": \"text\", \"text\": sample_example[\"prompt\"]}\n", + " ]\n", + " }\n", + " ]\n", + "]\n", + "\n", + "# 2. Process video and prompt messages into VLM inputs\n", + "base_inputs = prepare_smolvlm_inputs(\n", + " processor,\n", + " base_messages,\n", + " device,\n", + " video_fps=train_args.video_fps,\n", + " max_frames=train_args.max_frames,\n", + " video_size_longest_edge=train_args.video_size_longest_edge,\n", + " video_load_backend=train_args.video_load_backend\n", + ")\n", + "\n", + "# 3. Generate prediction using base SmolVLM2 model (visual context active)\n", + "print(\"Generating base model prediction (using visual tokens)...\")\n", + "base_generated_ids = raw_model.generate(\n", + " **base_inputs,\n", + " max_new_tokens=train_args.generation_max_new_tokens,\n", + " do_sample=False,\n", + " pad_token_id=tokenizer.pad_token_id,\n", + " eos_token_id=tokenizer.eos_token_id\n", + ")\n", + "\n", + "base_prediction = tokenizer.decode(\n", + " base_generated_ids[0][base_inputs[\"input_ids\"].shape[1]:],\n", + " skip_special_tokens=True\n", + ").strip()\n", + "\n", + "\n", + "# === Part B: Video2LoRA Inference (zero visual tokens) ===\n", + "\n", + "# 4. Inject the generated LoRA weights into the base language model layers\n", + "apply_lora_to_layers(\n", + " model.base_model,\n", + " model.hypernet.layer_indices,\n", + " generated_loras,\n", + " torch.ones(1, dtype=torch.int32, device=device),\n", + " position_ids=None\n", + ")\n", + "\n", + "# 5. Format a pure text-only prompt (zero visual tokens in context!)\n", + "prompt_ids = tokenizer.apply_chat_template(\n", + " [\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [{\"type\": \"text\", \"text\": sample_example[\"prompt\"]}]\n", + " }\n", + " ],\n", + " tokenize=True,\n", + " add_generation_prompt=True,\n", + " return_tensors=\"pt\"\n", + ").to(device)\n", + "\n", + "# 6. Generate prediction using Video2LoRA adapter-augmented model\n", + "print(\"Generating Video2LoRA prediction (zero visual tokens)...\")\n", + "generated_ids = model.base_model.generate(\n", + " input_ids=prompt_ids,\n", + " attention_mask=torch.ones_like(prompt_ids),\n", + " max_new_tokens=train_args.generation_max_new_tokens,\n", + " do_sample=False,\n", + " pad_token_id=tokenizer.pad_token_id,\n", + " eos_token_id=tokenizer.eos_token_id\n", + ")\n", + "\n", + "video2lora_prediction = tokenizer.decode(\n", + " generated_ids[0][prompt_ids.shape[1]:],\n", + " skip_special_tokens=True\n", + ").strip()\n", + "\n", + "# Clean up/reset the base model LoRA hooks\n", + "model.reset()\n", + "\n", + "print(\"\\n--- Outputs generated successfully. ---\")\n", + "print(f\"Base Model Output: {base_prediction}\")\n", + "print(f\"Video2LoRA Output: {video2lora_prediction}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 8: Qualitative Comparison Dashboard\n", + "\n", + "Display the comparison between the Base Model and Video2LoRA using the dynamically generated prediction outputs in a custom IPython HTML layout matching the evaluation layout." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from IPython.display import HTML, display\n", + "\n", + "def display_comparison(\n", + " image_url_or_path,\n", + " question_prompt,\n", + " ground_truth,\n", + " base_model_output,\n", + " video2lora_output\n", + "):\n", + " \"\"\"\n", + " Renders a beautifully styled comparison board matching the qualitative evaluation dashboard layout.\n", + " \"\"\"\n", + " html_content = f\"\"\"\n", + "
\n", + " \n", + " \n", + "
\n", + " \"Video\n", + "
\n", + " \n", + " \n", + "
\n", + "
QUESTION PROMPT
\n", + "

{question_prompt}

\n", + "
\n", + " \n", + " \n", + "
\n", + "
GROUND TRUTH
\n", + "

{ground_truth}

\n", + "
\n", + " \n", + " \n", + "
\n", + " \n", + " \n", + "
\n", + "
BASE MODEL
\n", + "

{base_model_output}

\n", + "
\n", + " \n", + " \n", + "
\n", + "
VIDEO2LORA
\n", + "

{video2lora_output}

\n", + "
\n", + " \n", + "
\n", + " \n", + "
\n", + " \"\"\"\n", + " display(HTML(html_content))\n", + "\n", + "# Display the comparison dashboard dynamically with model predictions!\n", + "display_comparison(\n", + " image_url_or_path=\"https://video2lora.github.io/assets/qualitative_watering.jpg\",\n", + " question_prompt=sample_example[\"prompt\"],\n", + " ground_truth=sample_example[\"target_text\"],\n", + " base_model_output=base_prediction,\n", + " video2lora_output=video2lora_prediction\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.0" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file