mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
Restructure notebook cells to run base model, video2lora, and visual rendering sequentially
This commit is contained in:
Sarvesh
parent
09b6068929
commit
a6d558b44d
1 changed files with 109 additions and 41 deletions
150
demo.ipynb
150
demo.ipynb
|
|
@ -556,11 +556,9 @@
|
|||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## \ud83d\udd0d Step 7: Run QA Inference (Base Model vs. Video2LoRA)\n",
|
||||
"## \ud83d\udd0d Step 7: Base Model Inference (With Visual Tokens)\n",
|
||||
"\n",
|
||||
"To evaluate the quality of the generated adapter, we perform a side-by-side comparison:\n",
|
||||
"- **Base Model (With Visual Tokens)**: We query the base model with the full visual video tokens in its context window (standard VLM execution).\n",
|
||||
"- **Video2LoRA (Zero Visual Tokens)**: We apply the hypernetwork-predicted LoRA weights to the model layers, clear out all visual tokens, and query the model using **text only**."
|
||||
"We first run inference using the **Base VLM Model** (with original video frames and visual tokens passed into the context window). We will query the base model for all 6 qualitative examples and save their predictions in a list `base_predictions`."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -569,14 +567,14 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"def run_inference(example, generated_loras, model, raw_model, processor, tokenizer, train_args, device):\n",
|
||||
" \"\"\"\n",
|
||||
" Runs inference comparing base model (using visual tokens) vs Video2LoRA (zero visual tokens).\n",
|
||||
" \"\"\"\n",
|
||||
" # === Part A: Base Model Inference (with visual tokens) ===\n",
|
||||
" # Ensure any patched LoRA hooks are cleared for a clean base VLM forward pass\n",
|
||||
" model.reset()\n",
|
||||
"# Ensure the model is reset to the clean base model state (LoRA hooks disabled)\n",
|
||||
"model.reset()\n",
|
||||
"\n",
|
||||
"base_predictions = []\n",
|
||||
"print(\"Running Base Model Inference (with visual tokens) for all 6 examples...\")\n",
|
||||
"for idx, example in enumerate(examples):\n",
|
||||
" print(f\"Processing Example {idx+1}/{len(examples)}: {example['id']}...\")\n",
|
||||
" \n",
|
||||
" base_messages = [\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
|
|
@ -607,15 +605,88 @@
|
|||
" eos_token_id=tokenizer.eos_token_id\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" base_prediction = tokenizer.decode(\n",
|
||||
" base_pred = tokenizer.decode(\n",
|
||||
" base_generated_ids[0][base_inputs[\"input_ids\"].shape[1]:],\n",
|
||||
" skip_special_tokens=True\n",
|
||||
" ).strip()\n",
|
||||
" \n",
|
||||
" base_predictions.append(base_pred)\n",
|
||||
"\n",
|
||||
" # === Part B: Video2LoRA Inference (zero visual tokens) ===\n",
|
||||
" # Re-patch the base model layers with the LoRA forward function before binding weights\n",
|
||||
"print(\"\\nBase Model inferences completed successfully.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## \u26a1 Step 8: Video2LoRA Internalization & Inference (Zero Visual Tokens)\n",
|
||||
"\n",
|
||||
"Next, we generate and run the dynamic adapter weights using **Video2LoRA** for all 6 qualitative examples. For each example:\n",
|
||||
"1. We run the visual frames through the base vision encoder to extract layer-by-layer activations.\n",
|
||||
"2. We pass the activations into the Perceiver Hypernetwork to generate the custom LoRA weights.\n",
|
||||
"3. We patch the base model's attention projections (`model.patch_lora_forward()`) and inject the generated adapter weights (`apply_lora_to_layers`).\n",
|
||||
"4. We query the VLM using **text only** (zero visual tokens in the context window) and save the outputs in `video2lora_predictions`."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"video2lora_predictions = []\n",
|
||||
"print(\"Running Video2LoRA Internalization & Inference (zero visual tokens) for all 6 examples...\")\n",
|
||||
"for idx, example in enumerate(examples):\n",
|
||||
" print(f\"Processing Example {idx+1}/{len(examples)}: {example['id']}...\")\n",
|
||||
" \n",
|
||||
" # 1. Reset model to ensure clean base features are extracted\n",
|
||||
" model.reset()\n",
|
||||
" \n",
|
||||
" # 2. Extract context activations\n",
|
||||
" internalize_messages = [\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"video\", \"path\": example[\"video_path\"]},\n",
|
||||
" {\"type\": \"text\", \"text\": train_args.internalization_prompt}\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
" ]\n",
|
||||
" ]\n",
|
||||
" \n",
|
||||
" vlm_inputs = prepare_smolvlm_inputs(\n",
|
||||
" processor,\n",
|
||||
" internalize_messages,\n",
|
||||
" device,\n",
|
||||
" video_fps=train_args.video_fps,\n",
|
||||
" max_frames=train_args.max_frames,\n",
|
||||
" video_size_longest_edge=train_args.video_size_longest_edge,\n",
|
||||
" video_load_backend=train_args.video_load_backend\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(\n",
|
||||
" raw_model,\n",
|
||||
" vlm_inputs,\n",
|
||||
" num_target_layers=model.hypernet.n_layers\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" # 3. Generate dynamic LoRA weights\n",
|
||||
" generated_loras, _ = model.generate_weights(\n",
|
||||
" ctx_ids=None,\n",
|
||||
" ctx_features=ctx_features,\n",
|
||||
" ctx_attn_mask=ctx_attn_mask,\n",
|
||||
" ctx_position_ids=ctx_position_ids\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" generated_loras = combine_lora(\n",
|
||||
" generated_loras,\n",
|
||||
" torch.ones(1, dtype=torch.int32, device=device),\n",
|
||||
" lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None\n",
|
||||
" )\n",
|
||||
" \n",
|
||||
" # 4. Patch base layers and inject generated weights\n",
|
||||
" model.patch_lora_forward()\n",
|
||||
"\n",
|
||||
" apply_lora_to_layers(\n",
|
||||
" model.base_model,\n",
|
||||
" model.hypernet.layer_indices,\n",
|
||||
|
|
@ -623,7 +694,8 @@
|
|||
" torch.ones(1, dtype=torch.int32, device=device),\n",
|
||||
" position_ids=None\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" # 5. Run text-only Q&A inference\n",
|
||||
" prompt_ids = tokenizer.apply_chat_template(\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
|
|
@ -636,7 +708,6 @@
|
|||
" return_tensors=\"pt\"\n",
|
||||
" ).to(device)\n",
|
||||
"\n",
|
||||
" # Handle cases where apply_chat_template returns a BatchEncoding dict instead of a raw Tensor\n",
|
||||
" if hasattr(prompt_ids, \"keys\"):\n",
|
||||
" input_ids = prompt_ids[\"input_ids\"]\n",
|
||||
" attention_mask = prompt_ids.get(\"attention_mask\", torch.ones_like(input_ids))\n",
|
||||
|
|
@ -644,7 +715,6 @@
|
|||
" input_ids = prompt_ids\n",
|
||||
" attention_mask = torch.ones_like(input_ids)\n",
|
||||
"\n",
|
||||
"\n",
|
||||
" generated_ids = model.base_model.generate(\n",
|
||||
" input_ids=input_ids,\n",
|
||||
" attention_mask=attention_mask,\n",
|
||||
|
|
@ -654,31 +724,26 @@
|
|||
" eos_token_id=tokenizer.eos_token_id\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" video2lora_prediction = tokenizer.decode(\n",
|
||||
" video2lora_pred = tokenizer.decode(\n",
|
||||
" generated_ids[0][input_ids.shape[1]:],\n",
|
||||
" skip_special_tokens=True\n",
|
||||
" ).strip()\n",
|
||||
" \n",
|
||||
" video2lora_predictions.append(video2lora_pred)\n",
|
||||
"\n",
|
||||
" # Reset model LoRA patches\n",
|
||||
" model.reset()\n",
|
||||
"# Reset model hooks back to clean base state\n",
|
||||
"model.reset()\n",
|
||||
"\n",
|
||||
" return base_prediction, video2lora_prediction\n",
|
||||
"\n",
|
||||
"# Run comparison inference on the sample example\n",
|
||||
"print(f\"Running inference for Example 2: {examples[1]['id']}...\")\n",
|
||||
"base_prediction, video2lora_prediction = run_inference(\n",
|
||||
" examples[1], sample_loras, model, raw_model, processor, tokenizer, train_args, device\n",
|
||||
")\n",
|
||||
"print(f\"\\nPredictions Generated:\\nBase Model: {base_prediction}\\nVideo2LoRA: {video2lora_prediction}\")"
|
||||
"print(\"\\nVideo2LoRA internalization and inferences completed successfully.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## \ud83c\udfa8 Step 8: Qualitative Comparison Dashboard\n",
|
||||
"## \ud83c\udfa8 Step 9: Qualitative Comparison Dashboard\n",
|
||||
"\n",
|
||||
"Finally, we run the internalization, inference, and evaluation loop for **all 6 qualitative examples** and render a custom HTML dashboard comparison matching the layout on the project website."
|
||||
"Below, we display the generated predictions. First, we print the text output summary directly to compare predictions side-by-side. Then, we render the beautiful visual qualitative comparison board for each example using our HTML template."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -747,21 +812,24 @@
|
|||
" \"\"\"\n",
|
||||
" display(HTML(html_content))\n",
|
||||
"\n",
|
||||
"# Loop through all 6 qualitative examples, run internalization + inference, and display the dashboards\n",
|
||||
"print(\"Processing and running qualitative comparison inference for all 6 examples...\")\n",
|
||||
"# Print summary text responses first\n",
|
||||
"print(\"=== Summary of Qualitative Responses ===\")\n",
|
||||
"for idx, example in enumerate(examples):\n",
|
||||
" print(f\"\\n--- Example {idx+1}: {example['id']} ({example['dataset']}) ---\")\n",
|
||||
" print(f\"Question: {example['prompt']}\")\n",
|
||||
" print(f\"Ground Truth: {example['target_text']}\")\n",
|
||||
" print(f\"Base Model : {base_predictions[idx]}\")\n",
|
||||
" print(f\"Video2LoRA : {video2lora_predictions[idx]}\")\n",
|
||||
"\n",
|
||||
"print(\"\\nRendering HTML Dashboards...\")\n",
|
||||
"# Display comparison boards\n",
|
||||
"for idx, example in enumerate(examples):\n",
|
||||
" print(f\"\\n--- Running Example {idx+1}/{len(examples)}: {example['id']} ---\")\n",
|
||||
" # 1. Internalize\n",
|
||||
" loras = run_internalization(example, model, raw_model, processor, train_args, device)\n",
|
||||
" # 2. Run inference\n",
|
||||
" base_pred, v2l_pred = run_inference(example, loras, model, raw_model, processor, tokenizer, train_args, device)\n",
|
||||
" # 3. Render dashboard\n",
|
||||
" display_comparison(\n",
|
||||
" video_path=example[\"video_path\"],\n",
|
||||
" question_prompt=example[\"prompt\"],\n",
|
||||
" ground_truth=example[\"target_text\"],\n",
|
||||
" base_model_output=base_pred,\n",
|
||||
" video2lora_output=v2l_pred,\n",
|
||||
" base_model_output=base_predictions[idx],\n",
|
||||
" video2lora_output=video2lora_predictions[idx],\n",
|
||||
" dataset_name=example[\"dataset\"]\n",
|
||||
" )"
|
||||
]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue