iclr 2026 (3)

webui/app.py

@@ -592,13 +592,38 @@ def chat():
                 ctx_tokenizer = None
                 with torch.inference_mode(), torch.amp.autocast(str(device)):
                     # Get the contexts and tokenize them
-                    contexts = request.form.getlist("contexts[]")
-                    if not contexts:
-                        contexts = [""]  # Use empty context if none provided
+                    raw_contexts = request.form.getlist("contexts[]")
+                    # Build scalers aligned with contexts (default to 1.0)
+                    raw_scalers = request.form.getlist("scalers[]")
+                    # Parse bias scaler (singular), default to 1.0 if missing/invalid
+                    bias_scaler_str = request.form.get("bias_scaler", "1.0")
+                    try:
+                        bias_scaler = float(bias_scaler_str)
+                    except Exception:
+                        bias_scaler = 1.0
 
-                    # Ensure we have at least one context
+                    # Clean contexts (drop empty), mirror selection for scalers
+                    pairs = list(zip(raw_contexts, raw_scalers))
+                    contexts = []
+                    kept_scalers = []
+                    for ctx, sc in pairs:
+                        if ctx.strip():
+                            contexts.append(ctx)
+                            try:
+                                kept_scalers.append(float(sc))
+                            except Exception:
+                                kept_scalers.append(1.0)
+
+                    # If all contexts are empty, use a single empty context and scaler 1.0
+                    if not contexts:
+                        contexts = [""]
+                        kept_scalers = [1.0]
+
+                    # Ensure we have at least one context (safety)
                     if len(contexts) == 1 and not contexts[0].strip():
                         contexts = [""]
+                        if not kept_scalers:
+                            kept_scalers = [1.0]
 
                     print(
                         f"Processing {len(contexts)} contexts for response generation"
@@ -621,17 +646,18 @@ def chat():
                     ctx_inputs = process_multiple_contexts(
                         contexts,
                         ctx_tokenizer,
-                        # max_length=(
-                        #     modulated_model.ctx_encoder_args.max_ctx_len
-                        #     if hasattr(modulated_model.ctx_encoder_args, "max_ctx_len")
-                        #     else 512
-                        # ),
                     )
 
                     ctx_ids = ctx_inputs["ctx_ids"].to(device)
                     ctx_attn_mask = ctx_inputs["ctx_attn_mask"].to(device)
 
+                    scalers_tensor = torch.tensor(
+                        kept_scalers, dtype=torch.float32, device=device
+                    )
+
                     print(f"chat_history: {chat_history}")
+                    print(f"scalers: {scalers_tensor}")
+                    print(f"bias_scaler: {bias_scaler}")
 
                     # Tokenize the chat history
                     model_inputs = base_tokenizer.apply_chat_template(
@@ -655,6 +681,8 @@ def chat():
                         n_ctx_chunks=torch.tensor(
                             [len(ctx_ids)], device=ctx_ids.device
                         ),
+                        scalers=scalers_tensor,  # pass per-context scalers
+                        bias_scaler=bias_scaler,  # pass singular bias scaler
                         input_ids=model_inputs,
                         max_new_tokens=512,
                         do_sample=False,

webui/templates/visualize.html

@@ -489,6 +489,21 @@
     .highlight-diff {
         background-color: #fff8e1;
     }
+
+    .context-scaler-controls {
+        display: flex;
+        gap: 10px;
+        align-items: center;
+        margin-top: 8px;
+    }
+
+    .context-scaler-controls input[type="range"] {
+        flex: 1;
+    }
+
+    .context-scaler-controls input[type="number"] {
+        width: 90px;
+    }
 </style>
 {% endblock %}
 
@@ -651,10 +666,23 @@
                         <label for="context-input-0">Context 1:</label>
                         <textarea id="context-input-0" class="context-input"
                             placeholder="Enter context information here. This will be used by the hypernetwork to generate a LoRA for the base model."></textarea>
+                        <label for="context-scaler-0" style="margin-top:8px;">Scaling:</label>
+                        <div class="context-scaler-controls">
+                            <input type="range" id="context-scaler-slider-0" class="context-scaler-slider" min="-2"
+                                max="2" step="0.01" value="1">
+                            <input type="number" id="context-scaler-0" class="context-scaler" step="0.01" value="1">
+                        </div>
                     </div>
                 </div>
             </div>
 
+            <!-- New singular Bias scaler control -->
+            <div class="context-field">
+                <label for="bias-scaler">Bias scaler:</label>
+                <input type="number" id="bias-scaler" step="0.01" value="1">
+                <p class="instruction-text">A single scalar applied to bias; independent of the number of contexts.</p>
+            </div>
+
             <button id="apply-hypernetwork-button">Apply Hypernetwork</button>
             <div id="hypernetwork-status"></div>
         </div>
@@ -806,14 +834,24 @@
                 let formData = new FormData();
                 formData.append('message', message);
 
-                // Check if we're using hypernetwork and include contexts
                 if (document.getElementById('chat-model-name').textContent.includes('Hypernetwork')) {
                     const contextInputs = document.querySelectorAll('.context-input');
-
-                    // Add all contexts to the form data
                     Array.from(contextInputs).forEach(input => {
                         formData.append('contexts[]', input.value);
                     });
+
+                    const scalerInputs = document.querySelectorAll('.context-scaler');
+                    Array.from(scalerInputs).forEach(input => {
+                        const v = (input.value || '').trim();
+                        formData.append('scalers[]', v.length ? v : '1.0');
+                    });
+
+                    // Include singular bias scaler
+                    const biasScalerInput = document.getElementById('bias-scaler');
+                    if (biasScalerInput) {
+                        const bv = (biasScalerInput.value || '').trim();
+                        formData.append('bias_scaler', bv.length ? bv : '1.0');
+                    }
                 }
 
                 fetch('/chat', {
@@ -845,6 +883,43 @@
             });
     }
 
+    // Utility to wire up slider-number sync inside a context-field
+    function attachScalerSync(fieldEl) {
+        const slider = fieldEl.querySelector('.context-scaler-slider');
+        const number = fieldEl.querySelector('.context-scaler');
+
+        if (!slider || !number) return;
+
+        // Keep slider within [-2,2], but allow number to be arbitrary
+        const clampToSlider = (x) => {
+            const min = parseFloat(slider.min || '-2');
+            const max = parseFloat(slider.max || '2');
+            if (isNaN(x)) return 1.0;
+            return Math.min(Math.max(x, min), max);
+        };
+
+        slider.addEventListener('input', () => {
+            number.value = slider.value;
+        });
+
+        number.addEventListener('input', () => {
+            const parsed = parseFloat(number.value);
+            if (isNaN(parsed)) {
+                number.value = '1.0';
+                slider.value = '1.0';
+            } else {
+                // do not clamp number; only clamp slider representation
+                slider.value = clampToSlider(parsed);
+            }
+        });
+    }
+
+    // Bind initial scaler sync
+    document.addEventListener('DOMContentLoaded', function () {
+        const firstField = document.querySelector('.context-fields .context-field');
+        if (firstField) attachScalerSync(firstField);
+    });
+
     // Handle hypernetwork checkpoint loading
     document.addEventListener('DOMContentLoaded', function () {
         const applyHypernetworkButton = document.getElementById('apply-hypernetwork-button');
@@ -1113,10 +1188,18 @@
                 <label for="context-input-${newFieldIndex}">Context ${contextCount}:</label>
                 <textarea id="context-input-${newFieldIndex}" class="context-input" 
                     placeholder="Enter additional context information here."></textarea>
+                <label for="context-scaler-${newFieldIndex}" style="margin-top:8px;">Scaling:</label>
+                <div class="context-scaler-controls">
+                    <input type="range" id="context-scaler-slider-${newFieldIndex}" class="context-scaler-slider" min="-2" max="2" step="0.01" value="1">
+                    <input type="number" id="context-scaler-${newFieldIndex}" class="context-scaler" step="0.01" value="1">
+                </div>
             `;
 
             contextFieldsContainer.appendChild(newField);
 
+            // Wire up slider-number sync for this field
+            attachScalerSync(newField);
+
             // Enable the remove button if we have more than one context
             if (contextCount > 1) {
                 removeContextButton.disabled = false;
@@ -1128,8 +1211,6 @@
             if (contextCount > 1) {
                 contextFieldsContainer.removeChild(contextFieldsContainer.lastChild);
                 contextCount--;
-
-                // Disable the remove button if we're back to just one context
                 if (contextCount === 1) {
                     removeContextButton.disabled = true;
                 }