mirror of
https://github.com/SakanaAI/doc-to-lora.git
synced 2026-07-23 17:01:04 +02:00
Support all 6 qualitative website examples inside demo.ipynb with auto video downloader and HTML5 video player
This commit is contained in:
Sarvesh
parent
fffc05f4e2
commit
482197ab9c
2 changed files with 208 additions and 449 deletions
376
demo.ipynb
376
demo.ipynb
|
|
@ -136,38 +136,72 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Define an example dataset manifest entry\n",
|
||||
"sample_example = {\n",
|
||||
" \"id\": \"sample-watering-plants\",\n",
|
||||
" \"video_path\": \"raw/finevideo/sample_watering_plants.mp4\",\n",
|
||||
" \"dataset\": \"finevideo\",\n",
|
||||
" \"task_type\": \"caption\",\n",
|
||||
" \"prompt\": \"Describe the key visible events in chronological order. Include all important actions and changes you can observe, with enough detail to distinguish each event clearly.\",\n",
|
||||
" \"target_text\": \"Little boy watering plants outdoors; Using watering can to pour water into flower pot; Shifting camera angle from side view to rear view; Tapping edge of flower pot a few times; Setting down watering can\"\n",
|
||||
"}\n",
|
||||
"# Define the 6 qualitative examples from the Video2LoRA project website\n",
|
||||
"examples = [\n",
|
||||
" {\n",
|
||||
" \"id\": \"man-smoking-pipe\",\n",
|
||||
" \"video_path\": \"media/benchmarks/carebench/v_00014063_0.mp4\",\n",
|
||||
" \"dataset\": \"CaReBench: Caption\",\n",
|
||||
" \"prompt\": \"Describe the video in as much useful visual detail as possible. Include the main activity, visible people or objects, scene context, appearance, and any important visual details that help explain what is happening.\",\n",
|
||||
" \"target_text\": \"This video depicts a scene of a man lighting a pipe with a lighter. The man in the video is smoking a pipe held in his mouth, supported by his left hand, while his right hand grips the lighter. His right forearm features a large black tattoo. He has short, thick hair that is a deep brown color and is dressed in a loose-fitting black tank top. He is seated next to a window, which has a wooden frame and blue curtains, with a brick wall behind him and a wooden door on the right. The door has a square pattern, adorned with silver hinges and a doorknob. In the video, he first ignites the lighter with his right hand and then brings the flame to the pipe, holding it in that position for several seconds. Throughout this time, his left hand remains steady on the pipe, and his gaze is fixed intently on it, ensuring that the pipe is fully lit before setting the lighter down. He then continues to hold the pipe with his left hand and begins to smoke. The video is shot from the front, clearly illustrating how relaxed he is while smoking at home.\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"id\": \"child-watering-plants\",\n",
|
||||
" \"video_path\": \"media/benchmarks/carebench/v_00016555_0.mp4\",\n",
|
||||
" \"dataset\": \"CaReBench: Events\",\n",
|
||||
" \"prompt\": \"Describe the key visible events in chronological order. Include all important actions and changes you can observe, with enough detail to distinguish each event clearly.\",\n",
|
||||
" \"target_text\": \"Little boy watering plants outdoors; Using watering can to pour water into flower pot; Shifting camera angle from side view to rear view; Tapping edge of flower pot a few times; Setting down watering can\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"id\": \"posture-alignment\",\n",
|
||||
" \"video_path\": \"media/benchmarks/plm/f522598789220c70_122_155.mp4\",\n",
|
||||
" \"dataset\": \"PLM-SGQA\",\n",
|
||||
" \"prompt\": \"Does this look like the same posture she's holding?\",\n",
|
||||
" \"target_text\": \"Yes, it appears you're mirroring the same posture. Your alignment, knee bend, and spine position match the demonstration, indicating proper form and engagement of the targeted muscle groups for optimal effectiveness and safety.\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"id\": \"dog-tugging\",\n",
|
||||
" \"video_path\": \"media/benchmarks/plm/b5bdb7f254cb1727_369_400.mp4\",\n",
|
||||
" \"dataset\": \"PLM-SGQA\",\n",
|
||||
" \"prompt\": \"Is he trying to tug?\",\n",
|
||||
" \"target_text\": \"Yes, your dog is likely inviting a tug-of-war game. Holding the toy in his mouth and possibly looking at you or wagging his tail indicates he's ready to engage in a playful tug.\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"id\": \"rainy-day\",\n",
|
||||
" \"video_path\": \"media/benchmarks/vidcapbench/132065802449.mp4\",\n",
|
||||
" \"dataset\": \"VidCapBench\",\n",
|
||||
" \"prompt\": \"What is the weather like in the scene? Answer only the question, in one sentence.\",\n",
|
||||
" \"target_text\": \"Rainy day.\"\n",
|
||||
" },\n",
|
||||
" {\n",
|
||||
" \"id\": \"tarsier-creature\",\n",
|
||||
" \"video_path\": \"media/benchmarks/vidcapbench/Tarsier_20.mp4\",\n",
|
||||
" \"dataset\": \"VidCapBench\",\n",
|
||||
" \"prompt\": \"Which parts of the creature are highlighted in the video? Answer only the question, in one sentence.\",\n",
|
||||
" \"target_text\": \"A close-up of its face, eyes, and hair.\"\n",
|
||||
" }\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"import os\n",
|
||||
"import urllib.request\n",
|
||||
"\n",
|
||||
"video_path = sample_example[\"video_path\"]\n",
|
||||
"if not os.path.exists(video_path):\n",
|
||||
" print(f\"Video file not found at: {video_path}\")\n",
|
||||
" print(\"Downloading a public sample video to run the demo...\")\n",
|
||||
"# Download the qualitative videos from the project website repo\n",
|
||||
"print(\"Checking/downloading qualitative benchmark video files from project website...\")\n",
|
||||
"for item in examples:\n",
|
||||
" video_path = item[\"video_path\"]\n",
|
||||
" os.makedirs(os.path.dirname(video_path), exist_ok=True)\n",
|
||||
" # Download a lightweight sample video (1.8MB)\n",
|
||||
" sample_url = \"https://commondatastorage.googleapis.com/gtv-videos-bucket/sample/ForBiggerBlazes.mp4\"\n",
|
||||
" try:\n",
|
||||
" urllib.request.urlretrieve(sample_url, video_path)\n",
|
||||
" print(f\"Sample video successfully downloaded to: {video_path}\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Failed to download sample video: {e}\")\n",
|
||||
" print(\"Please place a valid video at the path above.\")\n",
|
||||
"else:\n",
|
||||
" print(f\"Found video file at: {video_path}\")\n",
|
||||
" if not os.path.exists(video_path):\n",
|
||||
" url = f\"https://video2lora.github.io/{video_path}\"\n",
|
||||
" print(f\"Downloading {video_path} from {url}...\")\n",
|
||||
" try:\n",
|
||||
" urllib.request.urlretrieve(url, video_path)\n",
|
||||
" print(\"Download successful.\")\n",
|
||||
" except Exception as e:\n",
|
||||
" print(f\"Failed to download: {e}\")\n",
|
||||
" else:\n",
|
||||
" print(f\"Found local video file: {video_path}\")\n",
|
||||
"\n",
|
||||
"print(f\"\\nVideo ID: {sample_example['id']}\")\n",
|
||||
"print(f\"Prompt: {sample_example['prompt']}\")\n",
|
||||
"print(f\"Ground Truth: {sample_example['target_text']}\")"
|
||||
"print(f\"\\nLoaded {len(examples)} qualitative examples successfully.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -213,8 +247,9 @@
|
|||
" plt.show()\n",
|
||||
" cap.release()\n",
|
||||
"\n",
|
||||
"# To run the visualization, uncomment the line below:\n",
|
||||
"show_video_frames(sample_example[\"video_path\"], num_frames=4)"
|
||||
"# Visualize the first qualitative example (man smoking a pipe)\n",
|
||||
"print(f\"Visualizing keyframes for Example 1: {examples[0]['id']}\")\n",
|
||||
"show_video_frames(examples[0][\"video_path\"], num_frames=4)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -320,54 +355,56 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# 1. Define internalization messages\n",
|
||||
"internalize_messages = [\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"video\", \"path\": sample_example[\"video_path\"]},\n",
|
||||
" {\"type\": \"text\", \"text\": train_args.internalization_prompt}\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
"def run_internalization(example, model, raw_model, processor, train_args, device):\n",
|
||||
" \"\"\"\n",
|
||||
" Extracts visual features layer-by-layer and generates the dynamic LoRA adapter.\n",
|
||||
" \"\"\"\n",
|
||||
" internalize_messages = [\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"video\", \"path\": example[\"video_path\"]},\n",
|
||||
" {\"type\": \"text\", \"text\": train_args.internalization_prompt}\n",
|
||||
" ]\n }\n",
|
||||
" ]\n",
|
||||
" ]\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# 2. Process video and prompt messages into VLM input tensors\n",
|
||||
"vlm_inputs = prepare_smolvlm_inputs(\n",
|
||||
" processor,\n",
|
||||
" internalize_messages,\n",
|
||||
" device,\n",
|
||||
" video_fps=train_args.video_fps,\n",
|
||||
" max_frames=train_args.max_frames,\n",
|
||||
" video_size_longest_edge=train_args.video_size_longest_edge,\n",
|
||||
" video_load_backend=train_args.video_load_backend\n",
|
||||
")\n",
|
||||
" vlm_inputs = prepare_smolvlm_inputs(\n",
|
||||
" processor,\n",
|
||||
" internalize_messages,\n",
|
||||
" device,\n",
|
||||
" video_fps=train_args.video_fps,\n",
|
||||
" max_frames=train_args.max_frames,\n",
|
||||
" video_size_longest_edge=train_args.video_size_longest_edge,\n",
|
||||
" video_load_backend=train_args.video_load_backend\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# 3. Extract layer-by-layer hidden features from the vision encoder\n",
|
||||
"ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(\n",
|
||||
" raw_model,\n",
|
||||
" vlm_inputs,\n",
|
||||
" num_target_layers=model.hypernet.n_layers\n",
|
||||
")\n",
|
||||
" ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(\n",
|
||||
" raw_model,\n",
|
||||
" vlm_inputs,\n",
|
||||
" num_target_layers=model.hypernet.n_layers\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# 4. Generate custom LoRA adapter weights\n",
|
||||
"print(\"Generating dynamic LoRA weights...\")\n",
|
||||
"generated_loras, _ = model.generate_weights(\n",
|
||||
" ctx_ids=None,\n",
|
||||
" ctx_features=ctx_features,\n",
|
||||
" ctx_attn_mask=ctx_attn_mask,\n",
|
||||
" ctx_position_ids=ctx_position_ids\n",
|
||||
")\n",
|
||||
" generated_loras, _ = model.generate_weights(\n",
|
||||
" ctx_ids=None,\n",
|
||||
" ctx_features=ctx_features,\n",
|
||||
" ctx_attn_mask=ctx_attn_mask,\n",
|
||||
" ctx_position_ids=ctx_position_ids\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"# Combine generated adapter modules\n",
|
||||
"generated_loras = combine_lora(\n",
|
||||
" generated_loras,\n",
|
||||
" torch.ones(1, dtype=torch.int32, device=device),\n",
|
||||
" lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None\n",
|
||||
")\n",
|
||||
" generated_loras = combine_lora(\n",
|
||||
" generated_loras,\n",
|
||||
" torch.ones(1, dtype=torch.int32, device=device),\n",
|
||||
" lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
"print(\"LoRA adapter weights generated successfully.\")"
|
||||
" return generated_loras\n",
|
||||
"\n",
|
||||
"# Example run of internalization for child watering plants (index 1)\n",
|
||||
"print(f\"Running internalization for Example 2: {examples[1]['id']}...\")\n",
|
||||
"sample_loras = run_internalization(examples[1], model, raw_model, processor, train_args, device)\n",
|
||||
"print(\"Internalization successful.\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -385,94 +422,89 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# === Part A: Base Model Inference (with visual tokens) ===\n",
|
||||
"\n",
|
||||
"# 1. Format prompt for the base VLM (requires video inputs)\n",
|
||||
"base_messages = [\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"video\", \"path\": sample_example[\"video_path\"]},\n",
|
||||
" {\"type\": \"text\", \"text\": sample_example[\"prompt\"]}\n",
|
||||
" ]\n",
|
||||
" }\n",
|
||||
"def run_inference(example, generated_loras, model, raw_model, processor, tokenizer, train_args, device):\n",
|
||||
" \"\"\"\n",
|
||||
" Runs inference comparing base model (using visual tokens) vs Video2LoRA (zero visual tokens).\n",
|
||||
" \"\"\"\n",
|
||||
" # === Part A: Base Model Inference (with visual tokens) ===\n",
|
||||
" base_messages = [\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [\n",
|
||||
" {\"type\": \"video\", \"path\": example[\"video_path\"]},\n",
|
||||
" {\"type\": \"text\", \"text\": example[\"prompt\"]}\n",
|
||||
" ]\n }\n",
|
||||
" ]\n",
|
||||
" ]\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"# 2. Process video and prompt messages into VLM inputs\n",
|
||||
"base_inputs = prepare_smolvlm_inputs(\n",
|
||||
" processor,\n",
|
||||
" base_messages,\n",
|
||||
" device,\n",
|
||||
" video_fps=train_args.video_fps,\n",
|
||||
" max_frames=train_args.max_frames,\n",
|
||||
" video_size_longest_edge=train_args.video_size_longest_edge,\n",
|
||||
" video_load_backend=train_args.video_load_backend\n",
|
||||
" base_inputs = prepare_smolvlm_inputs(\n",
|
||||
" processor,\n",
|
||||
" base_messages,\n",
|
||||
" device,\n",
|
||||
" video_fps=train_args.video_fps,\n",
|
||||
" max_frames=train_args.max_frames,\n",
|
||||
" video_size_longest_edge=train_args.video_size_longest_edge,\n",
|
||||
" video_load_backend=train_args.video_load_backend\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" base_generated_ids = raw_model.generate(\n",
|
||||
" **base_inputs,\n",
|
||||
" max_new_tokens=train_args.generation_max_new_tokens,\n",
|
||||
" do_sample=False,\n",
|
||||
" pad_token_id=tokenizer.pad_token_id,\n",
|
||||
" eos_token_id=tokenizer.eos_token_id\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" base_prediction = tokenizer.decode(\n",
|
||||
" base_generated_ids[0][base_inputs[\"input_ids\"].shape[1]:],\n",
|
||||
" skip_special_tokens=True\n",
|
||||
" ).strip()\n",
|
||||
"\n",
|
||||
" # === Part B: Video2LoRA Inference (zero visual tokens) ===\n",
|
||||
" apply_lora_to_layers(\n",
|
||||
" model.base_model,\n",
|
||||
" model.hypernet.layer_indices,\n",
|
||||
" generated_loras,\n",
|
||||
" torch.ones(1, dtype=torch.int32, device=device),\n",
|
||||
" position_ids=None\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" prompt_ids = tokenizer.apply_chat_template(\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [{\"type\": \"text\", \"text\": example[\"prompt\"]}]\n }\n ],\n",
|
||||
" tokenize=True,\n",
|
||||
" add_generation_prompt=True,\n",
|
||||
" return_tensors=\"pt\"\n",
|
||||
" ).to(device)\n",
|
||||
"\n",
|
||||
" generated_ids = model.base_model.generate(\n",
|
||||
" input_ids=prompt_ids,\n",
|
||||
" attention_mask=torch.ones_like(prompt_ids),\n",
|
||||
" max_new_tokens=train_args.generation_max_new_tokens,\n",
|
||||
" do_sample=False,\n",
|
||||
" pad_token_id=tokenizer.pad_token_id,\n",
|
||||
" eos_token_id=tokenizer.eos_token_id\n",
|
||||
" )\n",
|
||||
"\n",
|
||||
" video2lora_prediction = tokenizer.decode(\n",
|
||||
" generated_ids[0][prompt_ids.shape[1]:],\n",
|
||||
" skip_special_tokens=True\n",
|
||||
" ).strip()\n",
|
||||
"\n",
|
||||
" # Reset model LoRA patches\n",
|
||||
" model.reset()\n",
|
||||
"\n",
|
||||
" return base_prediction, video2lora_prediction\n",
|
||||
"\n",
|
||||
"# Run comparison inference on the sample example\n",
|
||||
"print(f\"Running inference for Example 2: {examples[1]['id']}...\")\n",
|
||||
"base_prediction, video2lora_prediction = run_inference(\n",
|
||||
" examples[1], sample_loras, model, raw_model, processor, tokenizer, train_args, device\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 3. Generate prediction using base SmolVLM2 model (visual context active)\n",
|
||||
"print(\"Generating base model prediction (using visual tokens)...\")\n",
|
||||
"base_generated_ids = raw_model.generate(\n",
|
||||
" **base_inputs,\n",
|
||||
" max_new_tokens=train_args.generation_max_new_tokens,\n",
|
||||
" do_sample=False,\n",
|
||||
" pad_token_id=tokenizer.pad_token_id,\n",
|
||||
" eos_token_id=tokenizer.eos_token_id\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"base_prediction = tokenizer.decode(\n",
|
||||
" base_generated_ids[0][base_inputs[\"input_ids\"].shape[1]:],\n",
|
||||
" skip_special_tokens=True\n",
|
||||
").strip()\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"# === Part B: Video2LoRA Inference (zero visual tokens) ===\n",
|
||||
"\n",
|
||||
"# 4. Inject the generated LoRA weights into the base language model layers\n",
|
||||
"apply_lora_to_layers(\n",
|
||||
" model.base_model,\n",
|
||||
" model.hypernet.layer_indices,\n",
|
||||
" generated_loras,\n",
|
||||
" torch.ones(1, dtype=torch.int32, device=device),\n",
|
||||
" position_ids=None\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"# 5. Format a pure text-only prompt (zero visual tokens in context!)\n",
|
||||
"prompt_ids = tokenizer.apply_chat_template(\n",
|
||||
" [\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": [{\"type\": \"text\", \"text\": sample_example[\"prompt\"]}]\n",
|
||||
" }\n",
|
||||
" ],\n",
|
||||
" tokenize=True,\n",
|
||||
" add_generation_prompt=True,\n",
|
||||
" return_tensors=\"pt\"\n",
|
||||
").to(device)\n",
|
||||
"\n",
|
||||
"# 6. Generate prediction using Video2LoRA adapter-augmented model\n",
|
||||
"print(\"Generating Video2LoRA prediction (zero visual tokens)...\")\n",
|
||||
"generated_ids = model.base_model.generate(\n",
|
||||
" input_ids=prompt_ids,\n",
|
||||
" attention_mask=torch.ones_like(prompt_ids),\n",
|
||||
" max_new_tokens=train_args.generation_max_new_tokens,\n",
|
||||
" do_sample=False,\n",
|
||||
" pad_token_id=tokenizer.pad_token_id,\n",
|
||||
" eos_token_id=tokenizer.eos_token_id\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"video2lora_prediction = tokenizer.decode(\n",
|
||||
" generated_ids[0][prompt_ids.shape[1]:],\n",
|
||||
" skip_special_tokens=True\n",
|
||||
").strip()\n",
|
||||
"\n",
|
||||
"# Clean up/reset the base model LoRA hooks\n",
|
||||
"model.reset()\n",
|
||||
"\n",
|
||||
"print(\"\\n--- Outputs generated successfully. ---\")\n",
|
||||
"print(f\"Base Model Output: {base_prediction}\")\n",
|
||||
"print(f\"Video2LoRA Output: {video2lora_prediction}\")"
|
||||
"print(f\"\\nPredictions Generated:\\nBase Model: {base_prediction}\\nVideo2LoRA: {video2lora_prediction}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -493,21 +525,28 @@
|
|||
"from IPython.display import HTML, display\n",
|
||||
"\n",
|
||||
"def display_comparison(\n",
|
||||
" image_url_or_path,\n",
|
||||
" video_path,\n",
|
||||
" question_prompt,\n",
|
||||
" ground_truth,\n",
|
||||
" base_model_output,\n",
|
||||
" video2lora_output\n",
|
||||
" video2lora_output,\n",
|
||||
" dataset_name\n",
|
||||
"):\n",
|
||||
" \"\"\"\n",
|
||||
" Renders a beautifully styled comparison board matching the qualitative evaluation dashboard layout.\n",
|
||||
" Renders a beautifully styled comparison board with local HTML5 video player.\n",
|
||||
" \"\"\"\n",
|
||||
" html_content = f\"\"\"\n",
|
||||
" <div style=\"font-family: 'Segoe UI', Roboto, Helvetica, Arial, sans-serif; max-width: 900px; margin: 20px auto; color: #1e293b; background-color: #f8fafc; padding: 24px; border-radius: 16px; box-shadow: 0 4px 20px rgba(0,0,0,0.05);\">\n",
|
||||
" \n",
|
||||
" <!-- Video Frame Banner -->\n",
|
||||
" <div style=\"text-align: center; margin-bottom: 20px;\">\n",
|
||||
" <img src=\"{image_url_or_path}\" alt=\"Video Frame\" style=\"width: 100%; max-height: 480px; object-fit: cover; border-radius: 12px; box-shadow: 0 4px 12px rgba(0,0,0,0.1);\" onerror=\"this.onerror=null; this.src='https://images.unsplash.com/photo-1472289065668-ce650ac443d2?w=900&auto=format&fit=crop&q=80';\">\n",
|
||||
" <!-- Header Source Badge -->\n",
|
||||
" <div style=\"display: flex; justify-content: space-between; align-items: center; margin-bottom: 16px;\">\n",
|
||||
" <h3 style=\"margin: 0; font-size: 18px; font-weight: 700; color: #0f172a;\">Qualitative Comparison</h3>\n",
|
||||
" <span style=\"background-color: #e2e8f0; color: #475569; padding: 4px 10px; border-radius: 9999px; font-size: 12px; font-weight: 600;\">{dataset_name}</span>\n",
|
||||
" </div>\n",
|
||||
"\n",
|
||||
" <!-- Local HTML5 Video Player -->\n",
|
||||
" <div style=\"text-align: center; margin-bottom: 20px; background: #000; border-radius: 12px; overflow: hidden; box-shadow: 0 4px 12px rgba(0,0,0,0.15);\">\n",
|
||||
" <video src=\"{video_path}\" controls loop muted autoplay style=\"width: 100%; max-height: 400px; object-fit: contain; display: block; margin: 0 auto;\"></video>\n",
|
||||
" </div>\n",
|
||||
" \n",
|
||||
" <!-- Question Prompt Card -->\n",
|
||||
|
|
@ -527,13 +566,13 @@
|
|||
" \n",
|
||||
" <!-- Base Model Card -->\n",
|
||||
" <div style=\"background-color: #fff; border: 1.5px solid #ea580c; border-radius: 8px; padding: 16px; box-shadow: 0 2px 4px rgba(0,0,0,0.02);\">\n",
|
||||
" <h5 style=\"margin: 0 0 6px 0; color: #ea580c; text-transform: uppercase; font-size: 11px; letter-spacing: 0.05em; font-weight: 700;\">BASE MODEL</h5>\n",
|
||||
" <h5 style=\"margin: 0 0 6px 0; color: #ea580c; text-transform: uppercase; font-size: 11px; letter-spacing: 0.05em; font-weight: 700;\">BASE MODEL (with visual tokens)</h5>\n",
|
||||
" <p style=\"margin: 0; font-size: 14px; line-height: 1.5; color: #334155;\">{base_model_output}</p>\n",
|
||||
" </div>\n",
|
||||
" \n",
|
||||
" <!-- Video2LoRA Card -->\n",
|
||||
" <div style=\"background-color: #f0fdf4; border: 1.5px solid #16a34a; border-radius: 8px; padding: 16px; box-shadow: 0 2px 4px rgba(0,0,0,0.02);\">\n",
|
||||
" <h5 style=\"margin: 0 0 6px 0; color: #16a34a; text-transform: uppercase; font-size: 11px; letter-spacing: 0.05em; font-weight: 700;\">VIDEO2LORA</h5>\n",
|
||||
" <h5 style=\"margin: 0 0 6px 0; color: #16a34a; text-transform: uppercase; font-size: 11px; letter-spacing: 0.05em; font-weight: 700;\">VIDEO2LORA (zero visual tokens)</h5>\n",
|
||||
" <p style=\"margin: 0; font-size: 14px; line-height: 1.5; color: #1e293b; font-weight: 500;\">{video2lora_output}</p>\n",
|
||||
" </div>\n",
|
||||
" \n",
|
||||
|
|
@ -543,13 +582,14 @@
|
|||
" \"\"\"\n",
|
||||
" display(HTML(html_content))\n",
|
||||
"\n",
|
||||
"# Display the comparison dashboard dynamically with model predictions!\n",
|
||||
"# Display the comparison dashboard dynamically for our completed example!\n",
|
||||
"display_comparison(\n",
|
||||
" image_url_or_path=\"https://video2lora.github.io/assets/qualitative_watering.jpg\",\n",
|
||||
" question_prompt=sample_example[\"prompt\"],\n",
|
||||
" ground_truth=sample_example[\"target_text\"],\n",
|
||||
" video_path=examples[1][\"video_path\"],\n",
|
||||
" question_prompt=examples[1][\"prompt\"],\n",
|
||||
" ground_truth=examples[1][\"target_text\"],\n",
|
||||
" base_model_output=base_prediction,\n",
|
||||
" video2lora_output=video2lora_prediction\n",
|
||||
" video2lora_output=video2lora_prediction,\n",
|
||||
" dataset_name=examples[1][\"dataset\"]\n",
|
||||
")"
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,281 +0,0 @@
|
|||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
import torch
|
||||
|
||||
# Add repository root and src directory to sys.path to enable local imports
|
||||
repo_root = str(Path(__file__).parent.resolve())
|
||||
if repo_root not in sys.path:
|
||||
sys.path.insert(0, repo_root)
|
||||
src_dir = str(Path(repo_root) / "src")
|
||||
if os.path.exists(src_dir) and src_dir not in sys.path:
|
||||
sys.path.insert(0, src_dir)
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional
|
||||
from huggingface_hub import hf_hub_download
|
||||
|
||||
# Import core modules
|
||||
from ctx_to_lora.modeling.lora_layer import apply_lora_to_layers
|
||||
from ctx_to_lora.modeling.lora_merger import combine_lora
|
||||
from scripts.video2lora.train_smolvlm_stage1 import build_stage1_model
|
||||
from scripts.video2lora.train_smolvlm_online import (
|
||||
prepare_smolvlm_inputs,
|
||||
extract_l2l_fused_text_features,
|
||||
)
|
||||
|
||||
@dataclass
|
||||
class TrainArgs:
|
||||
smolvlm_name_or_path: str
|
||||
train_manifest: str = ""
|
||||
val_manifest: str = ""
|
||||
output_dir: str = ""
|
||||
lora_r: int = 16
|
||||
lora_dropout: float = 0.0
|
||||
target_modules: Optional[List[str]] = None
|
||||
latent_size: int = 512
|
||||
dropout_rate: float = 0.0
|
||||
n_latent_queries: int = 8
|
||||
num_blocks: int = 9
|
||||
num_self_attn_per_block: int = 0
|
||||
video_fps: Optional[float] = None
|
||||
max_frames: int = 12
|
||||
video_size_longest_edge: int = 384
|
||||
video_load_backend: str = "auto"
|
||||
internalization_prompt: str = "Internalize this video for later captioning."
|
||||
kl_weight: float = 0.0
|
||||
generation_max_new_tokens: int = 128
|
||||
|
||||
# 1. Qualitative Examples Database (from video2lora.github.io/script.js)
|
||||
examples = [
|
||||
{
|
||||
"videoId": "media/benchmarks/carebench/v_00014063_0.mp4",
|
||||
"question": "Describe the video in as much useful visual detail as possible. Include the main activity, visible people or objects, scene context, appearance, and any important visual details that help explain what is happening.",
|
||||
"reference": "This video depicts a scene of a man lighting a pipe with a lighter. The man in the video is smoking a pipe held in his mouth, supported by his left hand, while his right hand grips the lighter. His right forearm features a large black tattoo. He has short, thick hair that is a deep brown color and is dressed in a loose-fitting black tank top. He is seated next to a window, which has a wooden frame and blue curtains, with a brick wall behind him and a wooden door on the right. The door has a square pattern, adorned with silver hinges and a doorknob. In the video, he first ignites the lighter with his right hand and then brings the flame to the pipe, holding it in that position for several seconds. Throughout this time, his left hand remains steady on the pipe, and his gaze is fixed intently on it, ensuring that the pipe is fully lit before setting the lighter down. He then continues to hold the pipe with his left hand and begins to smoke. The video is shot from the front, clearly illustrating how relaxed he is while smoking at home.",
|
||||
"source": "CaReBench: Caption"
|
||||
},
|
||||
{
|
||||
"videoId": "media/benchmarks/carebench/v_00016555_0.mp4",
|
||||
"question": "Describe the key visible events in chronological order. Include all important actions and changes you can observe, with enough detail to distinguish each event clearly.",
|
||||
"reference": "Little boy watering plants outdoors; Using watering can to pour water into flower pot; Shifting camera angle from side view to rear view; Tapping edge of flower pot a few times; Setting down watering can",
|
||||
"source": "CaReBench: Events"
|
||||
},
|
||||
{
|
||||
"videoId": "media/benchmarks/plm/f522598789220c70_122_155.mp4",
|
||||
"question": "Does this look like the same posture she's holding?",
|
||||
"reference": "Yes, it appears you're mirroring the same posture. Your alignment, knee bend, and spine position match the demonstration, indicating proper form and engagement of the targeted muscle groups for optimal effectiveness and safety.",
|
||||
"source": "PLM-SGQA"
|
||||
},
|
||||
{
|
||||
"videoId": "media/benchmarks/plm/b5bdb7f254cb1727_369_400.mp4",
|
||||
"question": "Is he trying to tug?",
|
||||
"reference": "Yes, your dog is likely inviting a tug-of-war game. Holding the toy in his mouth and possibly looking at you or wagging his tail indicates he's ready to engage in a playful tug.",
|
||||
"source": "PLM-SGQA"
|
||||
},
|
||||
{
|
||||
"videoId": "media/benchmarks/vidcapbench/132065802449.mp4",
|
||||
"question": "What is the weather like in the scene? Answer only the question, in one sentence.",
|
||||
"reference": "Rainy day.",
|
||||
"source": "VidCapBench"
|
||||
},
|
||||
{
|
||||
"videoId": "media/benchmarks/vidcapbench/Tarsier_20.mp4",
|
||||
"question": "Which parts of the creature are highlighted in the video? Answer only the question, in one sentence.",
|
||||
"reference": "A close-up of its face, eyes, and hair.",
|
||||
"source": "VidCapBench"
|
||||
}
|
||||
]
|
||||
|
||||
def main():
|
||||
print("=== Video2LoRA: Qualitative Examples Inference Script ===")
|
||||
|
||||
# Download the qualitative videos if not present
|
||||
print("\n--- Downloading qualitative example videos ---")
|
||||
for item in examples:
|
||||
local_path = item["videoId"]
|
||||
os.makedirs(os.path.dirname(local_path), exist_ok=True)
|
||||
if not os.path.exists(local_path):
|
||||
url = f"https://video2lora.github.io/{item['videoId']}"
|
||||
print(f"Downloading {local_path} from {url}...")
|
||||
try:
|
||||
urllib.request.urlretrieve(url, local_path)
|
||||
print(f"Saved to {local_path}")
|
||||
except Exception as e:
|
||||
print(f"Error downloading {url}: {e}")
|
||||
else:
|
||||
print(f"Found local video file: {local_path}")
|
||||
|
||||
# Download hypernetwork weights
|
||||
print("\n--- Downloading 2.2B Video2LoRA checkpoint ---")
|
||||
checkpoint_dir = Path("checkpoints/Video2LoRA-SmolVLM-ckpts")
|
||||
checkpoint_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
ckpt_path = hf_hub_download(
|
||||
repo_id="MananSuri27/Video2LoRA-SmolVLM-ckpts",
|
||||
filename="video2lora-smolvlm2-2.2b-best-ce.pt",
|
||||
local_dir=str(checkpoint_dir)
|
||||
)
|
||||
print(f"Checkpoint ready at: {ckpt_path}")
|
||||
except Exception as e:
|
||||
print(f"Error downloading checkpoint: {e}")
|
||||
return
|
||||
|
||||
# Device selection (support MPS for Mac acceleration)
|
||||
device = torch.device("mps" if torch.backends.mps.is_available() else ("cuda" if torch.cuda.is_available() else "cpu"))
|
||||
print(f"\nTargeting device: {device}")
|
||||
|
||||
# Model parameters matching 2.2B SmolVLM2 preset
|
||||
train_args = TrainArgs(
|
||||
smolvlm_name_or_path="HuggingFaceTB/SmolVLM2-2.2B-Instruct",
|
||||
target_modules=["down_proj"]
|
||||
)
|
||||
|
||||
print("\nLoading SmolVLM2 base model and initializing modulated structure...")
|
||||
try:
|
||||
model, raw_model, processor, tokenizer = build_stage1_model(train_args, device=device)
|
||||
|
||||
print("Loading hypernetwork state dictionary...")
|
||||
state_dict = torch.load(ckpt_path, map_location="cpu")
|
||||
model.load_state_dict(state_dict)
|
||||
model.to(device)
|
||||
model.eval()
|
||||
raw_model.eval()
|
||||
print("Model initialization successful!")
|
||||
except Exception as e:
|
||||
print(f"Error loading models: {e}")
|
||||
return
|
||||
|
||||
# Loop and run inference for all examples
|
||||
for idx, item in enumerate(examples):
|
||||
print(f"\n==================================================")
|
||||
print(f"EXAMPLE {idx+1}/{len(examples)}: {item['source']}")
|
||||
print(f"==================================================")
|
||||
print(f"Video: {item['videoId']}")
|
||||
print(f"Question: {item['question']}")
|
||||
print(f"Ground Truth: {item['reference']}\n")
|
||||
|
||||
# 1. Perform internalization forward pass
|
||||
internalize_messages = [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video", "path": item["videoId"]},
|
||||
{"type": "text", "text": train_args.internalization_prompt}
|
||||
]
|
||||
}
|
||||
]
|
||||
]
|
||||
|
||||
with torch.no_grad():
|
||||
vlm_inputs = prepare_smolvlm_inputs(
|
||||
processor,
|
||||
internalize_messages,
|
||||
device,
|
||||
video_fps=train_args.video_fps,
|
||||
max_frames=train_args.max_frames,
|
||||
video_size_longest_edge=train_args.video_size_longest_edge,
|
||||
video_load_backend=train_args.video_load_backend
|
||||
)
|
||||
|
||||
ctx_features, ctx_attn_mask, ctx_position_ids = extract_l2l_fused_text_features(
|
||||
raw_model,
|
||||
vlm_inputs,
|
||||
num_target_layers=model.hypernet.n_layers
|
||||
)
|
||||
|
||||
generated_loras, _ = model.generate_weights(
|
||||
ctx_ids=None,
|
||||
ctx_features=ctx_features,
|
||||
ctx_attn_mask=ctx_attn_mask,
|
||||
ctx_position_ids=ctx_position_ids
|
||||
)
|
||||
|
||||
generated_loras = combine_lora(
|
||||
generated_loras,
|
||||
torch.ones(1, dtype=torch.int32, device=device),
|
||||
lora_bias=model.hypernet.get_head_bias() if model.hypernet.config.use_bias else None
|
||||
)
|
||||
|
||||
# 2. Run Base Model (with visual tokens in context)
|
||||
base_messages = [
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video", "path": item["videoId"]},
|
||||
{"type": "text", "text": item["question"]}
|
||||
]
|
||||
}
|
||||
]
|
||||
]
|
||||
|
||||
with torch.no_grad():
|
||||
base_inputs = prepare_smolvlm_inputs(
|
||||
processor,
|
||||
base_messages,
|
||||
device,
|
||||
video_fps=train_args.video_fps,
|
||||
max_frames=train_args.max_frames,
|
||||
video_size_longest_edge=train_args.video_size_longest_edge,
|
||||
video_load_backend=train_args.video_load_backend
|
||||
)
|
||||
|
||||
base_generated_ids = raw_model.generate(
|
||||
**base_inputs,
|
||||
max_new_tokens=train_args.generation_max_new_tokens,
|
||||
do_sample=False,
|
||||
pad_token_id=tokenizer.pad_token_id,
|
||||
eos_token_id=tokenizer.eos_token_id
|
||||
)
|
||||
|
||||
base_prediction = tokenizer.decode(
|
||||
base_generated_ids[0][base_inputs["input_ids"].shape[1]:],
|
||||
skip_special_tokens=True
|
||||
).strip()
|
||||
|
||||
# 3. Run Video2LoRA (zero visual tokens in context, using generated LoRA)
|
||||
apply_lora_to_layers(
|
||||
model.base_model,
|
||||
model.hypernet.layer_indices,
|
||||
generated_loras,
|
||||
torch.ones(1, dtype=torch.int32, device=device),
|
||||
position_ids=None
|
||||
)
|
||||
|
||||
prompt_ids = tokenizer.apply_chat_template(
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [{"type": "text", "text": item["question"]}]
|
||||
}
|
||||
],
|
||||
tokenize=True,
|
||||
add_generation_prompt=True,
|
||||
return_tensors="pt"
|
||||
).to(device)
|
||||
|
||||
with torch.no_grad():
|
||||
generated_ids = model.base_model.generate(
|
||||
input_ids=prompt_ids,
|
||||
attention_mask=torch.ones_like(prompt_ids),
|
||||
max_new_tokens=train_args.generation_max_new_tokens,
|
||||
do_sample=False,
|
||||
pad_token_id=tokenizer.pad_token_id,
|
||||
eos_token_id=tokenizer.eos_token_id
|
||||
)
|
||||
|
||||
video2lora_prediction = tokenizer.decode(
|
||||
generated_ids[0][prompt_ids.shape[1]:],
|
||||
skip_special_tokens=True
|
||||
).strip()
|
||||
|
||||
# Reset LoRA hooks
|
||||
model.reset()
|
||||
|
||||
print(f"--> Base Model: {base_prediction}")
|
||||
print(f"--> Video2LoRA: {video2lora_prediction}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Add table
Add a link
Reference in a new issue