From 4af1fe8638648c71d24c43b4f131e159c46270ab Mon Sep 17 00:00:00 2001 From: Sarvesh Date: Tue, 9 Jun 2026 20:56:38 +0530 Subject: [PATCH] Refactor tutorial notebook to extract utility functions and rename TrainArgs to Video2LoRAConfig --- demo.ipynb | 555 +++++++---------------------------------------- demo/__init__.py | 1 + demo/utils.py | 283 ++++++++++++++++++++++++ 3 files changed, 358 insertions(+), 481 deletions(-) create mode 100644 demo/__init__.py create mode 100644 demo/utils.py diff --git a/demo.ipynb b/demo.ipynb index c00b6d6..705f359 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -4,42 +4,18 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# \ud83c\udfa5 Video2LoRA: Self-Contained Zero-Visual-Token Inference Tutorial\n", + "# 🎥 Video2LoRA: Zero-Visual-Token Inference Tutorial\n", "\n", - "Welcome to the official interactive tutorial for **Video2LoRA**! \n", - "\n", - "This notebook provides a **completely self-contained, single-file walkthrough** demonstrating how to run zero-visual-token video inference. \n", - "\n", - "### \ud83e\udde0 The Core Concept: Parametric Video Internalization\n", - "Typically, Vision-Language Models (VLMs) process videos by projecting video frames into visual tokens and feeding hundreds (or thousands) of these tokens directly into the model's context window. This consumes massive context space, increases Time-to-First-Token (TTFT) latency, and slows down generation.\n", - "\n", - "**Video2LoRA** introduces a paradigm shift:\n", - "1. **Hypernetwork Generation**: A perceiver hypernetwork reads a video's intermediate activation features layer-by-layer and directly predicts a set of Low-Rank Adaptation (LoRA) weights in a single forward pass.\n", - "2. **Internalization**: Once generated, these adapter weights are injected into the base Vision-Language Model.\n", - "3. **Zero-Token Inference**: The VLM can now answer arbitrary text queries about the video **without requiring a single visual token** in its context window! The video information has been *internalized* parametrically into the model's weights.\n", - "\n", - "---\n", - "\n", - "### \ud83d\udd17 Project Resources\n", - "* **[\ud83c\udf10 Project Website](https://video2lora.github.io/)** \u2014 Learn more about the project, watch video demonstrations, and see qualitative benchmarks.\n", - "* **[\ud83d\udcc4 arXiv Paper](https://arxiv.org/abs/2606.04351)** \u2014 Read the full research paper detailing the architecture, training recipes, and evaluation.\n", - "* **[\ud83e\udd17 Hugging Face Checkpoints](https://huggingface.co/MananSuri27/Video2LoRA-SmolVLM-ckpts)** \u2014 Access raw pre-trained checkpoints for SmolVLM2.\n", - "* **[\ud83d\udcbb GitHub Repository](https://github.com/MananSuri27/video2lora)** \u2014 Explore the full codebase for training, evaluation, and data generation." + "Welcome to the official tutorial for **Video2LoRA**! This notebook demonstrates how to run zero-visual-token video inference. By using a perceiver hypernetwork, Video2LoRA reads video features layer-by-layer and predicts custom Low-Rank Adaptation (LoRA) weights. Once injected, the Vision-Language Model (VLM) can answer queries about the video **without requiring visual tokens** in its context window." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83d\udee0\ufe0f Step 0: Environment Setup & Package Installation\n", + "### Step 0: Environment Setup & Package Installation\n", "\n", - "In this step, we install the required open-source libraries and set up the directory structure. \n", - "\n", - "### \ud83d\udca1 Multi-Environment Compatibility\n", - "This cell automatically detects whether you are running in **Google Colab** or a **local Jupyter/VS Code** environment:\n", - "- **Google Colab**: It clones both the main `Video2LoRA` repository and the `video2lora.github.io` repository to retrieve the qualitative video files. It then sets up absolute path symlinks to resolve directories.\n", - "- **Local Environment**: If you have the repositories cloned locally in the same workspace, it automatically detects and symlinks the website's `media` folder to avoid redundant video downloads.\n", - "- **Dependencies**: Installs `transformers`, `peft`, `accelerate`, `decord`, `einops`, `jaxtyping`, and `num2words`." + "This cell automatically detects whether you are running in **Google Colab** or a **local workspace** and configures the environment accordingly." ] }, { @@ -48,7 +24,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Detect if running in Google Colab\n", + "# Detect Google Colab environment\n", "try:\n", " import google.colab\n", " IN_COLAB = True\n", @@ -58,32 +34,25 @@ "import os\n", "\n", "if IN_COLAB:\n", - " print(\"Running in Google Colab. Setting up workspace and website repositories...\")\n", - " # 1. Clone the main Video2LoRA repository to absolute path\n", + " print(\"Running in Google Colab. Setting up repositories...\")\n", " if not os.path.exists(\"/content/Video2LoRA\"):\n", " os.system(\"git clone -b demo https://github.com/video2lora/Video2LoRA.git /content/Video2LoRA\")\n", - " # 2. Clone the website repository to absolute path\n", " if not os.path.exists(\"/content/video2lora.github.io\"):\n", " os.system(\"git clone https://github.com/video2lora/video2lora.github.io.git /content/video2lora.github.io\")\n", - " # 3. Change directory to the codebase workspace\n", " os.chdir(\"/content/Video2LoRA\")\n", "\n", - "# Automatically set up the media symlink if the website repository is found nearby.\n", - "# This ensures that both local and Colab environments can resolve paths correctly.\n", + "# Link website media repository if available\n", "if os.path.lexists(\"media\"):\n", - " # If media exists but is a broken symlink or not a directory, remove it\n", " if not os.path.isdir(\"media\") or os.path.islink(\"media\"):\n", - " print(\"Removing conflicting/broken media symlink or file...\")\n", " try:\n", " if os.path.islink(\"media\"):\n", " os.unlink(\"media\")\n", " else:\n", " import shutil\n", " shutil.rmtree(\"media\")\n", - " except Exception as e:\n", + " except Exception:\n", " os.system(\"rm -rf media\")\n", "\n", - "# Dynamically find the website's media directory\n", "website_media_path = None\n", "possible_paths = [\n", " \"/content/video2lora.github.io/media\",\n", @@ -99,33 +68,23 @@ " break\n", "\n", "if website_media_path:\n", - " print(f\"Linking media directory from website repository at: {website_media_path}\")\n", " try:\n", " os.symlink(website_media_path, \"media\")\n", - " except Exception as e:\n", - " print(f\"Failed to create symlink using os.symlink, trying shell: {e}\")\n", + " except Exception:\n", " os.system(f\"ln -s {website_media_path} media\")\n", - "else:\n", - " print(\"Warning: Website media directory not found. Will download videos on demand.\")\n", "\n", "print(f\"Workspace directory: {os.getcwd()}\")\n", "\n", - "# Install necessary packages\n", + "# Install dependencies\n", "!pip install torch torchvision torchaudio\n", - "!pip install transformers accelerate peft huggingface_hub decord av opencv-python matplotlib einops jaxtyping num2words \"torchao>=0.16.0\"\n" + "!pip install transformers accelerate peft huggingface_hub decord av opencv-python matplotlib einops jaxtyping num2words \"torchao>=0.16.0\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83e\udde9 Step 1: Model Architectures & Utilities\n", - "\n", - "To ensure the tutorial is completely self-contained, we import the core architectures and utilities from the repository package. This includes:\n", - "- **LoRA Layer Injection**: `apply_lora_to_layers` from `ctx_to_lora.modeling.lora_layer` to patch the base model's attention projections.\n", - "- **LoRA Merger**: `combine_lora` from `ctx_to_lora.modeling.lora_merger` to combine weight matrices dynamically.\n", - "- **Model Construction**: `build_stage1_model` from `scripts.video2lora.train_smolvlm_stage1` to load and instantiate the modulated VLM.\n", - "- **VLM Preprocessing**: `prepare_smolvlm_inputs` and feature extraction helpers." + "### Step 1: Import Utilities & Patch Environment" ] }, { @@ -147,70 +106,29 @@ " sys.path.insert(0, src_dir)\n", "\n", "import torch\n", - "import cv2\n", - "import matplotlib.pyplot as plt\n", - "from dataclasses import dataclass\n", - "from typing import List, Optional\n", - "\n", - "# Import custom Video2LoRA architectures and utilities from the repository modules\n", - "from ctx_to_lora.modeling.lora_layer import apply_lora_to_layers\n", - "from ctx_to_lora.modeling.lora_merger import combine_lora\n", "from scripts.video2lora.train_smolvlm_stage1 import build_stage1_model\n", - "from scripts.video2lora.train_smolvlm_online import (\n", - " prepare_smolvlm_inputs,\n", - " extract_l2l_fused_text_features,\n", + "from scripts.video2lora.train_smolvlm_online import prepare_smolvlm_inputs\n", + "\n", + "# Import utilities from the demo package\n", + "from demo.utils import (\n", + " Video2LoRAConfig,\n", + " patch_num2words,\n", + " download_qualitative_videos,\n", + " show_video_frames,\n", + " run_internalization,\n", + " display_comparison,\n", + " apply_lora_to_layers\n", ")\n", "\n", - "@dataclass\n", - "class TrainArgs:\n", - " smolvlm_name_or_path: str\n", - " train_manifest: str\n", - " val_manifest: str\n", - " output_dir: str\n", - " lora_r: int = 16\n", - " lora_dropout: float = 0.0\n", - " target_modules: Optional[List[str]] = None\n", - " latent_size: int = 512\n", - " dropout_rate: float = 0.0\n", - " n_latent_queries: int = 8\n", - " num_blocks: int = 9\n", - " num_self_attn_per_block: int = 0\n", - " video_fps: Optional[float] = None\n", - " max_frames: int = 12\n", - " video_size_longest_edge: int = 384\n", - " video_load_backend: str = \"auto\"\n", - " internalization_prompt: str = \"Internalize this video for later captioning.\"\n", - " kl_weight: float = 0.0\n", - " generation_max_new_tokens: int = 128\n", - "\n", - "print(\"Video2LoRA imported architectures and utility functions successfully.\")\n", - "\n", - "# Force-refresh Hugging Face dependency cache for num2words if installed mid-session\n", - "import sys\n", - "import importlib.util\n", - "if importlib.util.find_spec(\"num2words\") is not None:\n", - " try:\n", - " from num2words import num2words as num2words_func\n", - " import transformers.utils.import_utils\n", - " transformers.utils.import_utils._num2words_available = True\n", - " for mod_name in list(sys.modules.keys()):\n", - " if \"processing_smolvlm\" in mod_name or \"smolvlm\" in mod_name:\n", - " mod = sys.modules[mod_name]\n", - " if hasattr(mod, \"num2words\"):\n", - " setattr(mod, \"num2words\", num2words_func)\n", - " except Exception:\n", - " pass\n" + "# Refresh Hugging Face dependency cache for num2words\n", + "patch_num2words()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83d\udcca Step 2: Load and Inspect Manifest\n", - "\n", - "Video2LoRA utilizes standard JSONL manifests to structure training and evaluation datasets. Below, we define the **6 qualitative examples** from the project page:\n", - "- **Examples included**: *Man smoking pipe*, *Child watering plants*, *Posture alignment*, *Dog tugging toy*, *Rainy day weather query*, and *Tarsier close-up*.\n", - "- **Local Video Retrieval**: The cell resolves directory paths recursively, handles broken symlinks automatically, and downloads the video files from the project website only if they are not already found locally." + "### Step 2: Define and Download Qualitative Examples" ] }, { @@ -219,7 +137,6 @@ "metadata": {}, "outputs": [], "source": [ - "# Define the 6 qualitative examples from the Video2LoRA project website\n", "examples = [\n", " {\n", " \"id\": \"man-smoking-pipe\",\n", @@ -263,71 +180,16 @@ " \"prompt\": \"Which parts of the creature are highlighted in the video? Answer only the question, in one sentence.\",\n", " \"target_text\": \"A close-up of its face, eyes, and hair.\"\n", " }\n", - "]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import urllib.request\n", + "]\n", "\n", - "# Download the qualitative videos from the project website repo\n", - "print(\"Checking/downloading qualitative benchmark video files from project website...\")\n", - "for item in examples:\n", - " video_path = item[\"video_path\"]\n", - " dir_name = os.path.dirname(video_path)\n", - " \n", - " # 1. Clean up any broken symlinks or conflicting files along the directories path\n", - " # to avoid FileNotFoundError during folder creation.\n", - " parts = video_path.split('/')\n", - " current_path = \"\"\n", - " for part in parts[:-1]: # skip the filename\n", - " if current_path:\n", - " current_path = os.path.join(current_path, part)\n", - " else:\n", - " current_path = part\n", - " if os.path.lexists(current_path) and not os.path.isdir(current_path):\n", - " print(f\"Removing conflict at '{current_path}' (broken link or file) to allow directory creation...\")\n", - " try:\n", - " if os.path.islink(current_path):\n", - " os.unlink(current_path)\n", - " else:\n", - " os.remove(current_path)\n", - " except Exception as e:\n", - " print(f\"Failed to remove conflict at {current_path}: {e}\")\n", - " \n", - " # 2. Safely create directory structure\n", - " try:\n", - " os.makedirs(dir_name, exist_ok=True)\n", - " except Exception as e:\n", - " print(f\"Warning: Could not create directory {dir_name}: {e}\")\n", - " \n", - " # 3. Check and retrieve the video file\n", - " if not os.path.exists(video_path):\n", - " url = f\"https://video2lora.github.io/{video_path}\"\n", - " print(f\"Downloading {video_path} from {url}...\")\n", - " try:\n", - " urllib.request.urlretrieve(url, video_path)\n", - " print(\"Download successful.\")\n", - " except Exception as e:\n", - " print(f\"Failed to download: {e}\")\n", - " else:\n", - " print(f\"Found local video file: {video_path}\")\n", - "\n", - "print(f\"\\nLoaded {len(examples)} qualitative examples successfully.\")" + "download_qualitative_videos(examples)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83d\uddbc\ufe0f Step 3: Visualize Video Frames\n", - "\n", - "Before running inference, let's write a utility function using OpenCV and Matplotlib to extract keyframes from the video file and inspect the visual input. We'll visualize the first example: **Man smoking a pipe**." + "### Step 3: Visualize Video Frames" ] }, { @@ -336,36 +198,7 @@ "metadata": {}, "outputs": [], "source": [ - "def show_video_frames(video_path, num_frames=4):\n", - " \"\"\"\n", - " Helper function to load keyframes from a video path and plot them in a grid.\n", - " \"\"\"\n", - " if not os.path.exists(video_path):\n", - " print(f\"Video file not found at: {video_path} (Please provide a valid video to inspect).\")\n", - " return\n", - " \n", - " cap = cv2.VideoCapture(video_path)\n", - " total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))\n", - " indices = [int(i * total_frames / num_frames) for i in range(num_frames)]\n", - " \n", - " fig, axes = plt.subplots(1, num_frames, figsize=(16, 4))\n", - " for i, idx in enumerate(indices):\n", - " cap.set(cv2.CAP_PROP_POS_FRAMES, idx)\n", - " ret, frame = cap.read()\n", - " if ret:\n", - " frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)\n", - " axes[i].imshow(frame)\n", - " axes[i].axis('off')\n", - " axes[i].set_title(f\"Frame {idx}\")\n", - " else:\n", - " axes[i].text(0.5, 0.5, \"Frame Read Error\", ha=\"center\", va=\"center\")\n", - " axes[i].axis('off')\n", - " plt.tight_layout()\n", - " plt.show()\n", - " cap.release()\n", - "\n", - "# Visualize the first qualitative example (man smoking a pipe)\n", - "print(f\"Visualizing keyframes for Example 1: {examples[0]['id']}\")\n", + "# Visualize keyframes for the first qualitative example\n", "show_video_frames(examples[0][\"video_path\"], num_frames=4)" ] }, @@ -373,9 +206,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83d\udcbe Step 4: Download Video2LoRA Checkpoint\n", - "\n", - "We use the Hugging Face hub client to download the pre-trained weights for the **2.2B SmolVLM2 hypernetwork** (`video2lora-smolvlm2-2.2b-best-ce.pt`). This checkpoint is trained on FineVideo using Cross-Entropy teacher distillation." + "### Step 4: Download Video2LoRA Checkpoint" ] }, { @@ -384,21 +215,17 @@ "metadata": {}, "outputs": [], "source": [ - "from pathlib import Path\n", "from huggingface_hub import hf_hub_download\n", "\n", - "print(\"Downloading 2.2B Video2LoRA Checkpoint from Hugging Face...\")\n", - "\n", + "print(\"Downloading pre-trained 2.2B Video2LoRA Checkpoint...\")\n", "checkpoint_dir = Path(\"checkpoints/Video2LoRA-SmolVLM-ckpts\")\n", "checkpoint_dir.mkdir(parents=True, exist_ok=True)\n", "\n", - "# Download checkpoint weights programmatically\n", "ckpt_path = hf_hub_download(\n", " repo_id=\"MananSuri27/Video2LoRA-SmolVLM-ckpts\",\n", " filename=\"video2lora-smolvlm2-2.2b-best-ce.pt\",\n", " local_dir=str(checkpoint_dir)\n", ")\n", - "\n", "print(f\"Checkpoint successfully downloaded to: {ckpt_path}\")" ] }, @@ -406,12 +233,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83d\ude80 Step 5: Load Base VLM & Initialize Modulated Model\n", - "\n", - "We configure `TrainArgs` matching the SmolVLM2 2.2B evaluation preset and:\n", - "1. Initialize the base SmolVLM2 vision-language model (`HuggingFaceTB/SmolVLM2-2.2B-Instruct`), processor, and tokenizer.\n", - "2. Load the pre-trained perceiver hypernetwork weights.\n", - "3. Put the model in evaluation mode (`eval()`) and move it to the target hardware device (CUDA GPU if available, otherwise CPU)." + "### Step 5: Initialize SmolVLM2 & Video2LoRA Models" ] }, { @@ -420,8 +242,8 @@ "metadata": {}, "outputs": [], "source": [ - "# Set configuration arguments for the 2.2B SmolVLM2 model preset\n", - "train_args = TrainArgs(\n", + "# Set configuration arguments matching the 2.2B SmolVLM2 model preset\n", + "config = Video2LoRAConfig(\n", " smolvlm_name_or_path=\"HuggingFaceTB/SmolVLM2-2.2B-Instruct\",\n", " train_manifest=\"\",\n", " val_manifest=\"\",\n", @@ -443,41 +265,17 @@ " generation_max_new_tokens=128\n", ")\n", "\n", - "# Configure device\n", "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n", "\n", - "# 1. Initialize SmolVLM2 base model, processor, and tokenizer\n", - "print(\"Loading SmolVLM2 model (2.2B-Instruct)...\")\n", + "print(\"Loading SmolVLM2 model...\")\n", + "model, raw_model, processor, tokenizer = build_stage1_model(config, device=device)\n", "\n", - "\n", - "# Force-refresh Hugging Face dependency cache for num2words if installed mid-session\n", - "import sys\n", - "import importlib.util\n", - "if importlib.util.find_spec(\"num2words\") is not None:\n", - " try:\n", - " from num2words import num2words as num2words_func\n", - " import transformers.utils.import_utils\n", - " transformers.utils.import_utils._num2words_available = True\n", - " for mod_name in list(sys.modules.keys()):\n", - " if \"processing_smolvlm\" in mod_name or \"smolvlm\" in mod_name:\n", - " mod = sys.modules[mod_name]\n", - " if hasattr(mod, \"num2words\"):\n", - " setattr(mod, \"num2words\", num2words_func)\n", - " except Exception:\n", - " pass\n", - "\n", - "model, raw_model, processor, tokenizer = build_stage1_model(train_args, device=device)\n", - "\n", - "# 2. Load the hypernetwork weights\n", "print(\"Loading hypernetwork weights...\")\n", "state_dict = torch.load(ckpt_path, map_location=\"cpu\", weights_only=False)\n", "model.load_state_dict(state_dict)\n", "\n", - "# Place models in evaluation mode\n", - "model.to(device)\n", - "model.eval()\n", + "model.to(device).eval()\n", "raw_model.eval()\n", - "\n", "print(\"Model load complete.\")" ] }, @@ -485,9 +283,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## \u26a1 Step 6: Parametric Video Internalization\n", - "\n", - "This is where the core Video2LoRA magic happens! We feed the video frames through the visual encoder to extract intermediate activations layer-by-layer. Then, the perceiver hypernetwork processes these activations and generates the custom LoRA adapter weights in a single pass." + "### Step 6: Test Video Internalization" ] }, { @@ -496,69 +292,17 @@ "metadata": {}, "outputs": [], "source": [ - "def run_internalization(example, model, raw_model, processor, train_args, device):\n", - " \"\"\"\n", - " Extracts visual features layer-by-layer and generates the dynamic LoRA adapter.\n", - " \"\"\"\n", - " # Ensure any patched LoRA hooks are cleared before feature extraction\n", - " model.reset()\n", - "\n", - " internalize_messages = [\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"video\", \"path\": example[\"video_path\"]},\n", - " {\"type\": \"text\", \"text\": train_args.internalization_prompt}\n", - " ]\n", - " }\n", - " ]\n", - " ]\n", - "\n", - " vlm_inputs = prepare_smolvlm_inputs(\n", - " processor,\n", - " internalize_messages,\n", - " device,\n", - " video_fps=train_args.video_fps,\n", - " max_frames=train_args.max_frames,\n", - " video_size_longest_edge=train_args.video_size_longest_edge,\n", - " video_load_backend=train_args.video_load_backend\n", - " )\n", - "\n", - " ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(\n", - " raw_model,\n", - " vlm_inputs,\n", - " num_target_layers=model.hypernet.n_layers\n", - " )\n", - "\n", - " generated_loras, _ = model.generate_weights(\n", - " ctx_ids=None,\n", - " ctx_features=ctx_features,\n", - " ctx_attn_mask=ctx_attn_mask,\n", - " ctx_position_ids=ctx_position_ids\n", - " )\n", - "\n", - " generated_loras = combine_lora(\n", - " generated_loras,\n", - " torch.ones(1, dtype=torch.int32, device=device),\n", - " lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None\n", - " )\n", - "\n", - " return generated_loras\n", - "\n", - "# Example run of internalization for child watering plants (index 1)\n", + "# Run internalization for Example 2\n", "print(f\"Running internalization for Example 2: {examples[1]['id']}...\")\n", - "sample_loras = run_internalization(examples[1], model, raw_model, processor, train_args, device)\n", - "print(\"Internalization successful.\")" + "sample_loras = run_internalization(examples[1], model, raw_model, processor, config, device)\n", + "print(\"Internalization test successful.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83d\udd0d Step 7: Base Model Inference (With Visual Tokens)\n", - "\n", - "We first run inference using the **Base VLM Model** (with original video frames and visual tokens passed into the context window). We will query the base model for all 6 qualitative examples and save their predictions in a list `base_predictions`." + "### Step 7: Base Model Inference (With Visual Tokens)" ] }, { @@ -567,46 +311,43 @@ "metadata": {}, "outputs": [], "source": [ - "import torch\n", "import gc\n", "\n", - "# Ensure the model is reset to the clean base model state (LoRA hooks disabled)\n", + "# Reset model state\n", "model.reset()\n", "torch.cuda.empty_cache()\n", "gc.collect()\n", "\n", "base_predictions = []\n", - "print(\"Running Base Model Inference (with visual tokens) for all 6 examples...\")\n", + "print(\"Running Base Model Inference (with visual tokens) for all examples...\")\n", "\n", "with torch.inference_mode():\n", " for idx, example in enumerate(examples):\n", " print(f\"Processing Example {idx+1}/{len(examples)}: {example['id']}...\")\n", " \n", - " base_messages = [\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"video\", \"path\": example[\"video_path\"]},\n", - " {\"type\": \"text\", \"text\": example[\"prompt\"]}\n", - " ]\n", - " }\n", - " ]\n", - " ]\n", + " base_messages = [[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": [\n", + " {\"type\": \"video\", \"path\": example[\"video_path\"]},\n", + " {\"type\": \"text\", \"text\": example[\"prompt\"]}\n", + " ]\n", + " }\n", + " ]]\n", "\n", " base_inputs = prepare_smolvlm_inputs(\n", " processor,\n", " base_messages,\n", " device,\n", - " video_fps=train_args.video_fps,\n", - " max_frames=train_args.max_frames,\n", - " video_size_longest_edge=train_args.video_size_longest_edge,\n", - " video_load_backend=train_args.video_load_backend\n", + " video_fps=config.video_fps,\n", + " max_frames=config.max_frames,\n", + " video_size_longest_edge=config.video_size_longest_edge,\n", + " video_load_backend=config.video_load_backend\n", " )\n", "\n", " base_generated_ids = raw_model.generate(\n", " **base_inputs,\n", - " max_new_tokens=train_args.generation_max_new_tokens,\n", + " max_new_tokens=config.generation_max_new_tokens,\n", " do_sample=False,\n", " pad_token_id=tokenizer.pad_token_id,\n", " eos_token_id=tokenizer.eos_token_id\n", @@ -619,25 +360,18 @@ " \n", " base_predictions.append(base_pred)\n", " \n", - " # Clean up temporary GPU tensors for this step\n", " del base_inputs, base_generated_ids\n", " torch.cuda.empty_cache()\n", " gc.collect()\n", "\n", - "print(\"\\nBase Model inferences completed successfully.\")" + "print(\"\\nBase Model inferences complete.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \u26a1 Step 8: Video2LoRA Internalization & Inference (Zero Visual Tokens)\n", - "\n", - "Next, we generate and run the dynamic adapter weights using **Video2LoRA** for all 6 qualitative examples. For each example:\n", - "1. We run the visual frames through the base vision encoder to extract layer-by-layer activations.\n", - "2. We pass the activations into the Perceiver Hypernetwork to generate the custom LoRA weights.\n", - "3. We patch the base model's attention projections (`model.patch_lora_forward()`) and inject the generated adapter weights (`apply_lora_to_layers`).\n", - "4. We query the VLM using **text only** (zero visual tokens in the context window) and save the outputs in `video2lora_predictions`." + "### Step 8: Video2LoRA Internalization & Inference (Zero Visual Tokens)" ] }, { @@ -646,65 +380,17 @@ "metadata": {}, "outputs": [], "source": [ - "import torch\n", - "import gc\n", - "\n", "video2lora_predictions = []\n", - "print(\"Running Video2LoRA Internalization & Inference (zero visual tokens) for all 6 examples...\")\n", + "print(\"Running Video2LoRA Inference (zero visual tokens) for all examples...\")\n", "\n", "with torch.inference_mode():\n", " for idx, example in enumerate(examples):\n", " print(f\"Processing Example {idx+1}/{len(examples)}: {example['id']}...\")\n", " \n", - " # 1. Reset model to ensure clean base features are extracted\n", - " model.reset()\n", - " torch.cuda.empty_cache()\n", - " gc.collect()\n", + " # Generate custom LoRA adapter\n", + " generated_loras = run_internalization(example, model, raw_model, processor, config, device)\n", " \n", - " # 2. Extract context activations\n", - " internalize_messages = [\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [\n", - " {\"type\": \"video\", \"path\": example[\"video_path\"]},\n", - " {\"type\": \"text\", \"text\": train_args.internalization_prompt}\n", - " ]\n", - " }\n", - " ]\n", - " ]\n", - " \n", - " vlm_inputs = prepare_smolvlm_inputs(\n", - " processor,\n", - " internalize_messages,\n", - " device,\n", - " video_fps=train_args.video_fps,\n", - " max_frames=train_args.max_frames,\n", - " video_size_longest_edge=train_args.video_size_longest_edge,\n", - " video_load_backend=train_args.video_load_backend\n", - " )\n", - "\n", - " ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(\n", - " raw_model,\n", - " vlm_inputs,\n", - " num_target_layers=model.hypernet.n_layers\n", - " )\n", - "\n", - " # 3. Generate dynamic LoRA weights\n", - " generated_loras, _ = model.generate_weights(\n", - " ctx_ids=None,\n", - " ctx_features=ctx_features,\n", - " ctx_attn_mask=ctx_attn_mask,\n", - " ctx_position_ids=ctx_position_ids\n", - " )\n", - "\n", - " generated_loras = combine_lora(\n", - " generated_loras,\n", - " torch.ones(1, dtype=torch.int32, device=device),\n", - " lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None\n", - " )\n", - " \n", - " # 4. Patch base layers and inject generated weights\n", + " # Inject dynamic weights\n", " model.patch_lora_forward()\n", " apply_lora_to_layers(\n", " model.base_model,\n", @@ -714,14 +400,12 @@ " position_ids=None\n", " )\n", " \n", - " # 5. Run text-only Q&A inference\n", + " # Query VLM using text only (zero visual tokens)\n", " prompt_ids = tokenizer.apply_chat_template(\n", - " [\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": [{\"type\": \"text\", \"text\": example[\"prompt\"]}]\n", - " }\n", - " ],\n", + " [{\n", + " \"role\": \"user\",\n", + " \"content\": [{\"type\": \"text\", \"text\": example[\"prompt\"]}]\n", + " }],\n", " tokenize=True,\n", " add_generation_prompt=True,\n", " return_tensors=\"pt\"\n", @@ -737,7 +421,7 @@ " generated_ids = model.base_model.generate(\n", " input_ids=input_ids,\n", " attention_mask=attention_mask,\n", - " max_new_tokens=train_args.generation_max_new_tokens,\n", + " max_new_tokens=config.generation_max_new_tokens,\n", " do_sample=False,\n", " pad_token_id=tokenizer.pad_token_id,\n", " eos_token_id=tokenizer.eos_token_id\n", @@ -750,27 +434,22 @@ " \n", " video2lora_predictions.append(video2lora_pred)\n", " \n", - " # Clean up temporary GPU tensors for this step\n", - " del vlm_inputs, ctx_features, ctx_attn_mask, ctx_position_ids, generated_loras, prompt_ids, input_ids, attention_mask, generated_ids\n", + " del generated_loras, prompt_ids, input_ids, attention_mask, generated_ids\n", " model.reset()\n", " torch.cuda.empty_cache()\n", " gc.collect()\n", "\n", - "# Reset model hooks back to clean base state\n", "model.reset()\n", "torch.cuda.empty_cache()\n", "gc.collect()\n", - "\n", - "print(\"\\nVideo2LoRA internalization and inferences completed successfully.\")" + "print(\"\\nVideo2LoRA inferences complete.\")" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## \ud83c\udfa8 Step 9: Qualitative Comparison Dashboard\n", - "\n", - "Below, we display the generated predictions. First, we print the text output summary directly to compare predictions side-by-side. Then, we render the beautiful visual qualitative comparison board for each example using our HTML template." + "### Step 9: Qualitative Comparison Dashboard" ] }, { @@ -779,91 +458,6 @@ "metadata": {}, "outputs": [], "source": [ - "from IPython.display import HTML, display\n", - "import base64\n", - "import os\n", - "\n", - "def get_video_base64(video_path):\n", - " \"\"\"\n", - " Reads a local video file and encodes it to a base64 data URI to play in sandboxed Colab iframes.\n", - " \"\"\"\n", - " if not os.path.exists(video_path):\n", - " return \"\"\n", - " try:\n", - " with open(video_path, \"rb\") as f:\n", - " data = f.read()\n", - " b64_str = base64.b64encode(data).decode(\"utf-8\")\n", - " return f\"data:video/mp4;base64,{b64_str}\"\n", - " except Exception as e:\n", - " print(f\"Warning: Failed to encode video {video_path} to base64: {e}\")\n", - " return \"\"\n", - "\n", - "def display_comparison(\n", - " video_path,\n", - " question_prompt,\n", - " ground_truth,\n", - " base_model_output,\n", - " video2lora_output,\n", - " dataset_name\n", - "):\n", - " \"\"\"\n", - " Renders a beautifully styled comparison board with local HTML5 video player.\n", - " \"\"\"\n", - " video_src = get_video_base64(video_path)\n", - " if not video_src:\n", - " if video_path.startswith((\"http://\", \"https://\")):\n", - " video_src = video_path\n", - " else:\n", - " video_src = f\"https://video2lora.github.io/{video_path}\"\n", - " \n", - " html_content = f\"\"\"\n", - "
\n", - " \n", - " \n", - "
\n", - "

Qualitative Comparison

\n", - " {dataset_name}\n", - "
\n", - "\n", - " \n", - "
\n", - " \n", - "
\n", - " \n", - " \n", - "
\n", - "
QUESTION PROMPT
\n", - "

{question_prompt}

\n", - "
\n", - " \n", - " \n", - "
\n", - "
GROUND TRUTH
\n", - "

{ground_truth}

\n", - "
\n", - " \n", - " \n", - "
\n", - " \n", - " \n", - "
\n", - "
BASE MODEL (with visual tokens)
\n", - "

{base_model_output}

\n", - "
\n", - " \n", - " \n", - "
\n", - "
VIDEO2LORA (zero visual tokens)
\n", - "

{video2lora_output}

\n", - "
\n", - " \n", - "
\n", - " \n", - "
\n", - " \"\"\"\n", - " display(HTML(html_content))\n", - "\n", - "# Print summary text responses first\n", "print(\"=== Summary of Qualitative Responses ===\")\n", "for idx, example in enumerate(examples):\n", " print(f\"\\n--- Example {idx+1}: {example['id']} ({example['dataset']}) ---\")\n", @@ -873,7 +467,6 @@ " print(f\"Video2LoRA : {video2lora_predictions[idx]}\")\n", "\n", "print(\"\\nRendering HTML Dashboards...\")\n", - "# Display comparison boards\n", "for idx, example in enumerate(examples):\n", " display_comparison(\n", " video_path=example[\"video_path\"],\n", diff --git a/demo/__init__.py b/demo/__init__.py new file mode 100644 index 0000000..f2256fa --- /dev/null +++ b/demo/__init__.py @@ -0,0 +1 @@ +# Video2LoRA Demo Package diff --git a/demo/utils.py b/demo/utils.py new file mode 100644 index 0000000..237f81f --- /dev/null +++ b/demo/utils.py @@ -0,0 +1,283 @@ +import os +import sys +import base64 +import urllib.request +from dataclasses import dataclass +from typing import List, Optional +from pathlib import Path + +# Add repository root and src directory to sys.path to enable local imports +repo_root = str(Path(__file__).resolve().parent.parent) +if repo_root not in sys.path: + sys.path.insert(0, repo_root) +src_dir = str(Path(repo_root) / "src") +if os.path.exists(src_dir) and src_dir not in sys.path: + sys.path.insert(0, src_dir) + +import torch +import cv2 +import matplotlib.pyplot as plt +from IPython.display import HTML, display + +# Local repository imports +from ctx_to_lora.modeling.lora_layer import apply_lora_to_layers +from ctx_to_lora.modeling.lora_merger import combine_lora +from scripts.video2lora.train_smolvlm_online import ( + prepare_smolvlm_inputs, + extract_l2l_fused_text_features, +) + + +@dataclass +class Video2LoRAConfig: + smolvlm_name_or_path: str + train_manifest: str + val_manifest: str + output_dir: str + lora_r: int = 16 + lora_dropout: float = 0.0 + target_modules: Optional[List[str]] = None + latent_size: int = 512 + dropout_rate: float = 0.0 + n_latent_queries: int = 8 + num_blocks: int = 9 + num_self_attn_per_block: int = 0 + video_fps: Optional[float] = None + max_frames: int = 12 + video_size_longest_edge: int = 384 + video_load_backend: str = "auto" + internalization_prompt: str = "Internalize this video for later captioning." + kl_weight: float = 0.0 + generation_max_new_tokens: int = 128 + + +def patch_num2words(): + """ + Force-refresh Hugging Face dependency cache for num2words if installed mid-session. + """ + import importlib.util + if importlib.util.find_spec("num2words") is not None: + try: + from num2words import num2words as num2words_func + import transformers.utils.import_utils + transformers.utils.import_utils._num2words_available = True + for mod_name in list(sys.modules.keys()): + if "processing_smolvlm" in mod_name or "smolvlm" in mod_name: + mod = sys.modules[mod_name] + if hasattr(mod, "num2words"): + setattr(mod, "num2words", num2words_func) + except Exception: + pass + + + +def download_qualitative_videos(examples): + """ + Download the qualitative videos from the project website repo if not found locally. + """ + print("Checking/downloading qualitative benchmark video files from project website...") + for item in examples: + video_path = item["video_path"] + dir_name = os.path.dirname(video_path) + + # 1. Clean up any broken symlinks or conflicting files along the directories path + # to avoid FileNotFoundError during folder creation. + parts = video_path.split('/') + current_path = "" + for part in parts[:-1]: # skip the filename + if current_path: + current_path = os.path.join(current_path, part) + else: + current_path = part + if os.path.lexists(current_path) and not os.path.isdir(current_path): + print(f"Removing conflict at '{current_path}' (broken link or file) to allow directory creation...") + try: + if os.path.islink(current_path): + os.unlink(current_path) + else: + os.remove(current_path) + except Exception as e: + print(f"Failed to remove conflict at {current_path}: {e}") + + # 2. Safely create directory structure + try: + os.makedirs(dir_name, exist_ok=True) + except Exception as e: + print(f"Warning: Could not create directory {dir_name}: {e}") + + # 3. Check and retrieve the video file + if not os.path.exists(video_path): + url = f"https://video2lora.github.io/{video_path}" + print(f"Downloading {video_path} from {url}...") + try: + urllib.request.urlretrieve(url, video_path) + print("Download successful.") + except Exception as e: + print(f"Failed to download: {e}") + else: + print(f"Found local video file: {video_path}") + + print(f"\nLoaded {len(examples)} qualitative examples successfully.") + + +def show_video_frames(video_path, num_frames=4): + """ + Helper function to load keyframes from a video path and plot them in a grid. + """ + if not os.path.exists(video_path): + print(f"Video file not found at: {video_path} (Please provide a valid video to inspect).") + return + + cap = cv2.VideoCapture(video_path) + total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) + indices = [int(i * total_frames / num_frames) for i in range(num_frames)] + + fig, axes = plt.subplots(1, num_frames, figsize=(16, 4)) + for i, idx in enumerate(indices): + cap.set(cv2.CAP_PROP_POS_FRAMES, idx) + ret, frame = cap.read() + if ret: + frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + axes[i].imshow(frame) + axes[i].axis('off') + axes[i].set_title(f"Frame {idx}") + else: + axes[i].text(0.5, 0.5, "Frame Read Error", ha="center", va="center") + axes[i].axis('off') + plt.tight_layout() + plt.show() + cap.release() + + +def run_internalization(example, model, raw_model, processor, config, device): + """ + Extracts visual features layer-by-layer and generates the dynamic LoRA adapter. + """ + # Ensure any patched LoRA hooks are cleared before feature extraction + model.reset() + + internalize_messages = [ + [ + { + "role": "user", + "content": [ + {"type": "video", "path": example["video_path"]}, + {"type": "text", "text": config.internalization_prompt} + ] + } + ] + ] + + vlm_inputs = prepare_smolvlm_inputs( + processor, + internalize_messages, + device, + video_fps=config.video_fps, + max_frames=config.max_frames, + video_size_longest_edge=config.video_size_longest_edge, + video_load_backend=config.video_load_backend + ) + + + ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features( + raw_model, + vlm_inputs, + num_target_layers=model.hypernet.n_layers + ) + + generated_loras, _ = model.generate_weights( + ctx_ids=None, + ctx_features=ctx_features, + ctx_attn_mask=ctx_attn_mask, + ctx_position_ids=ctx_position_ids + ) + + generated_loras = combine_lora( + generated_loras, + torch.ones(1, dtype=torch.int32, device=device), + lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None + ) + + return generated_loras + + +def get_video_base64(video_path): + """ + Reads a local video file and encodes it to a base64 data URI to play in sandboxed Colab iframes. + """ + if not os.path.exists(video_path): + return "" + try: + with open(video_path, "rb") as f: + data = f.read() + b64_str = base64.b64encode(data).decode("utf-8") + return f"data:video/mp4;base64,{b64_str}" + except Exception as e: + print(f"Warning: Failed to encode video {video_path} to base64: {e}") + return "" + + +def display_comparison( + video_path, + question_prompt, + ground_truth, + base_model_output, + video2lora_output, + dataset_name +): + """ + Renders a beautifully styled comparison board with local HTML5 video player. + """ + video_src = get_video_base64(video_path) + if not video_src: + if video_path.startswith(("http://", "https://")): + video_src = video_path + else: + video_src = f"https://video2lora.github.io/{video_path}" + + html_content = f""" +
+ + +
+

Qualitative Comparison

+ {dataset_name} +
+ + +
+ +
+ + +
+
QUESTION PROMPT
+

{question_prompt}

+
+ + +
+
GROUND TRUTH
+

{ground_truth}

+
+ + +
+ + +
+
BASE MODEL (with visual tokens)
+

{base_model_output}

+
+ + +
+
VIDEO2LORA (zero visual tokens)
+

{video2lora_output}

+
+ +
+ +
+ """ + display(HTML(html_content))