feat: refactor node spec and add mcp tools (#244)

* refactor: carve out extraction panel

* refactor: create spec versions for node types

* refactor: create a GenericNode and remove custom nodes

* feat: add python and typescript sdk

* add dograh sdk

* fix: fetch draft workflow definition over published one

* fix: fix routes of SDKs to use code gen

* chore: remove doclink dependency to reduce image size

* chore: format files

* chore: bump pipecat

* feat: let mcp fetch archived workflows on demand

* chore: fix tests

* feat: add sdk documentation

* chore: change banner and add badge

ui/src/app/files/DocumentUpload.tsx

@@ -1,6 +1,6 @@
 'use client';
 
-import { FileText, Upload, X } from 'lucide-react';
+import { FileText, Info, Upload, X } from 'lucide-react';
 import { useRef, useState } from 'react';
 import { toast } from 'sonner';
 
@@ -13,6 +13,7 @@ import { Button } from '@/components/ui/button';
 import { Label } from '@/components/ui/label';
 import { Progress } from '@/components/ui/progress';
 import { RadioGroup, RadioGroupItem } from '@/components/ui/radio-group';
+import { useAppConfig } from '@/context/AppConfigContext';
 import logger from '@/lib/logger';
 
 interface DocumentUploadProps {
@@ -23,6 +24,8 @@ const MAX_FILE_SIZE = 5 * 1024 * 1024; // 5MB
 const ACCEPTED_FILE_TYPES = ['.pdf', '.docx', '.doc', '.txt', '.json'];
 
 export default function DocumentUpload({ onUploadSuccess }: DocumentUploadProps) {
+  const { config } = useAppConfig();
+  const isOSS = config?.deploymentMode === 'oss';
   const [selectedFile, setSelectedFile] = useState<File | null>(null);
   const [retrievalMode, setRetrievalMode] = useState<string>('full_document');
   const [uploading, setUploading] = useState(false);
@@ -30,6 +33,21 @@ export default function DocumentUpload({ onUploadSuccess }: DocumentUploadProps)
   const [dragActive, setDragActive] = useState(false);
   const fileInputRef = useRef<HTMLInputElement>(null);
 
+  const ossNotice = isOSS ? (
+    <div className="flex gap-3 rounded-lg border border-amber-200 bg-amber-50 p-3 dark:border-amber-900/50 dark:bg-amber-950/30">
+      <Info className="h-4 w-4 flex-shrink-0 text-amber-600 dark:text-amber-400 mt-0.5" />
+      <div className="text-xs text-amber-900 dark:text-amber-200">
+        <p className="font-medium">Processed by an external service</p>
+        <p className="mt-1">
+          Uploaded documents are sent to Dograh&apos;s managed Model Proxy Service for
+          parsing and chunking. Dograh Model Proxy Service does not store or read your documents -
+          the extracted text and embeddings are returned and stored locally in your
+          self-hosted database.
+        </p>
+      </div>
+    </div>
+  ) : null;
+
   const validateFile = (file: File): boolean => {
     const fileExtension = '.' + file.name.split('.').pop()?.toLowerCase();
     if (!ACCEPTED_FILE_TYPES.includes(fileExtension)) {
@@ -164,6 +182,7 @@ export default function DocumentUpload({ onUploadSuccess }: DocumentUploadProps)
   if (selectedFile && !uploading) {
     return (
       <div className="space-y-4">
+        {ossNotice}
         {/* Selected file info */}
         <div className="flex items-center gap-3 p-3 border rounded-lg bg-muted/30">
           <FileText className="w-8 h-8 text-primary flex-shrink-0" />
@@ -225,6 +244,7 @@ export default function DocumentUpload({ onUploadSuccess }: DocumentUploadProps)
 
   return (
     <div className="space-y-4">
+      {ossNotice}
       <input
         ref={fileInputRef}
         type="file"

ui/src/app/workflow/[workflowId]/RenderWorkflow.tsx

@@ -19,7 +19,7 @@ import { WorkflowConfigurations } from '@/types/workflow-configurations';
 
 import AddNodePanel from "../../../components/flow/AddNodePanel";
 import CustomEdge from "../../../components/flow/edges/CustomEdge";
-import { AgentNode, EndCall, GlobalNode, QANode, StartCall, TriggerNode, WebhookNode } from "../../../components/flow/nodes";
+import { GenericNode } from "../../../components/flow/nodes/GenericNode";
 import { PhoneCallDialog } from './components/PhoneCallDialog';
 import { VersionHistoryPanel, WorkflowVersion } from './components/VersionHistoryPanel';
 import { WorkflowEditorHeader } from "./components/WorkflowEditorHeader";
@@ -27,16 +27,13 @@ import { WorkflowProvider } from "./contexts/WorkflowContext";
 import { useWorkflowState } from "./hooks/useWorkflowState";
 import { layoutNodes } from './utils/layoutNodes';
 
-// Define the node types dynamically based on the onSave prop
-const nodeTypes = {
-    [NodeType.START_CALL]: StartCall,
-    [NodeType.AGENT_NODE]: AgentNode,
-    [NodeType.END_CALL]: EndCall,
-    [NodeType.GLOBAL_NODE]: GlobalNode,
-    [NodeType.TRIGGER]: TriggerNode,
-    [NodeType.WEBHOOK]: WebhookNode,
-    [NodeType.QA]: QANode,
-};
+// Single generic component for every node type. The spec catalog
+// (`/api/v1/node-types`) drives form rendering, canvas preview, handles,
+// and defaults. Adding a new node type means adding a Python NodeSpec —
+// no React changes required.
+const nodeTypes = Object.fromEntries(
+    Object.values(NodeType).map((t) => [t, GenericNode]),
+);
 
 const edgeTypes = {
     custom: CustomEdge,

ui/src/app/workflow/[workflowId]/hooks/useWorkflowState.ts

@@ -17,141 +17,54 @@ import {
     updateWorkflowApiV1WorkflowWorkflowIdPut,
     validateWorkflowApiV1WorkflowWorkflowIdValidatePost
 } from "@/client";
-import { WorkflowError } from "@/client/types.gen";
-import { FlowEdge, FlowNode, NodeType } from "@/components/flow/types";
+import { NodeSpec, WorkflowError } from "@/client/types.gen";
+import { useNodeSpecs } from "@/components/flow/renderer";
+import { FlowEdge, FlowNode, FlowNodeData, NodeType } from "@/components/flow/types";
 import { PostHogEvent } from "@/constants/posthog-events";
 import logger from '@/lib/logger';
 import { getNextNodeId, getRandomId } from "@/lib/utils";
 import { DEFAULT_WORKFLOW_CONFIGURATIONS, WorkflowConfigurations } from "@/types/workflow-configurations";
 
-const DEFAULT_QA_SYSTEM_PROMPT = `You are a QA analyst evaluating a specific segment of a voice AI conversation.
-
-## Node Purpose
-{{node_summary}}
-
-## Previous Conversation Context (For start of conversation, previous conversation summary can be empty.)
-{{previous_conversation_summary}}
-
-## Tags to evaluate
-
-Examine the conversation carefully and identify which of the following tags apply:
-
-- UNCLEAR_CONVERSATION - The conversation is not coherent or clear, messages don't connect logically
-- ASSISTANT_IN_LOOP - The assistant asks the same question multiple times or gets stuck repeating itself
-- ASSISTANT_REPLY_IMPROPER - The assistant did not reply properly to the user's question/query or seems confused by what the user said
-- USER_FRUSTRATED - The user seems angry, frustrated, or is complaining about something in the call
-- USER_NOT_UNDERSTANDING - The user explicitly says they don't understand or repeatedly asks for clarification
-- HEARING_ISSUES - Either party can't hear the other ("hello?", "are you there?", "can you hear me?")
-- DEAD_AIR - Unusually long silences in the conversation (use the timestamps to judge)
-- USER_REQUESTING_FEATURE - The user asks for something the assistant can't fulfill
-- ASSISTANT_LACKS_EMPATHY - The assistant ignores the user's personal situation or emotional state and continues pitching or pushing the agenda.
-- USER_DETECTS_AI - The user suspects or identifies that they are talking to an AI/robot/bot rather than a real human.
-
-## Call metrics (pre-computed)
-
-Use these alongside the transcript for your analysis:
-{{metrics}}
-
-## Output format
-
-Return ONLY a valid JSON object (no markdown):
-{
-    "tags": [
-        {
-            "tag": "TAG_NAME",
-            "reason": "Short reason with evidence from the transcript"
+// Build initial node data from spec defaults. Replaces the per-type
+// hardcoded `getNewNode` switch — adding a new node type is now zero
+// frontend code: declare the spec on the backend and the defaults flow
+// through here.
+function buildDataFromSpec(spec: NodeSpec): Record<string, unknown> {
+    const data: Record<string, unknown> = {};
+    for (const prop of spec.properties) {
+        if (prop.default !== undefined && prop.default !== null) {
+            data[prop.name] = prop.default;
         }
-    ],
-    "overall_sentiment": "positive|neutral|negative",
-    "call_quality_score": <1-10>,
-    "summary": "1-2 sentence summary of this segment"
-}
-
-If no tags apply, return an empty tags list. Always provide sentiment, score, and summary.`;
-
-export function getDefaultAllowInterrupt(type: string = NodeType.START_CALL): boolean {
-    switch (type) {
-        case NodeType.AGENT_NODE:
-            return true; // Agents can be interrupted
-        case NodeType.START_CALL:
-        case NodeType.END_CALL:
-            return false; // Start/End messages should not be interrupted
-        default:
-            return false;
     }
+    return data;
 }
 
-const defaultNodes: FlowNode[] = [
-    {
-        id: "1",
-        type: NodeType.START_CALL,
-        position: { x: 200, y: 200 },
-        data: {
-            prompt: "",
-            name: "",
-            allow_interrupt: getDefaultAllowInterrupt(NodeType.START_CALL),
-        },
-    },
-];
-
-const getNewNode = (type: string, position: { x: number, y: number }, existingNodes: FlowNode[]) => {
-    // Base node configuration
-    const baseNode = {
+function buildNewNode(
+    type: string,
+    position: { x: number; y: number },
+    existingNodes: FlowNode[],
+    spec: NodeSpec,
+): FlowNode {
+    const data = buildDataFromSpec(spec) as Partial<FlowNodeData> & Record<string, unknown>;
+    if (type === NodeType.START_CALL) data.is_start = true;
+    if (type === NodeType.END_CALL) data.is_end = true;
+    return {
         id: getNextNodeId(existingNodes),
         type,
         position,
-        data: {
-            prompt: {
-                [NodeType.GLOBAL_NODE]: "You are a helpful assistant whose mode of interaction with the user is voice. So don't use any special characters which can not be pronounced. Use short sentences and simple language.",
-            }[type] || "",
-            name: {
-                [NodeType.GLOBAL_NODE]: "Global Node",
-                [NodeType.START_CALL]: "Start Call",
-                [NodeType.END_CALL]: "End Call",
-                [NodeType.WEBHOOK]: "Webhook",
-                [NodeType.QA]: "QA Analysis",
-            }[type] || "",
-            allow_interrupt: getDefaultAllowInterrupt(type),
-        },
+        data: data as unknown as FlowNode["data"],
     };
+}
 
-    // Add webhook-specific defaults
-    if (type === NodeType.WEBHOOK) {
-        return {
-            ...baseNode,
-            data: {
-                ...baseNode.data,
-                enabled: true,
-                http_method: "POST" as const,
-                endpoint_url: "",
-                custom_headers: [],
-                payload_template: {
-                    call_id: "{{workflow_run_id}}",
-                    first_name: "{{initial_context.first_name}}",
-                    rsvp: "{{gathered_context.rsvp}}",
-                    duration: "{{cost_info.call_duration_seconds}}",
-                    recording_url: "{{recording_url}}",
-                    transcript_url: "{{transcript_url}}",
-                },
-            },
-        };
-    }
-
-    // Add QA-specific defaults
-    if (type === NodeType.QA) {
-        return {
-            ...baseNode,
-            data: {
-                ...baseNode.data,
-                qa_enabled: true,
-                qa_model: "default",
-                qa_system_prompt: DEFAULT_QA_SYSTEM_PROMPT,
-            },
-        };
-    }
-
-    return baseNode;
-};
+// Look up the spec default for `allow_interrupt`. Used as a load-time
+// fallback for older saved workflows whose nodes lack the field.
+function specAllowInterrupt(
+    type: string,
+    bySpecName: Map<string, NodeSpec>,
+): boolean | undefined {
+    const prop = bySpecName.get(type)?.properties.find((p) => p.name === "allow_interrupt");
+    return prop?.default as boolean | undefined;
+}
 
 interface UseWorkflowStateProps {
     initialWorkflowName: string;
@@ -181,6 +94,10 @@ export const useWorkflowState = ({
     const router = useRouter();
     const rfInstance = useRef<ReactFlowInstance<FlowNode, FlowEdge> | null>(null);
 
+    // Spec catalog. Workflow init waits on this to populate defaults; node
+    // creation looks up per-type schemas through it.
+    const { bySpecName, loading: specsLoading } = useNodeSpecs();
+
     // Get state and actions from the store
     const {
         nodes,
@@ -214,20 +131,32 @@ export const useWorkflowState = ({
     const canUndo = useWorkflowStore((state) => state.canUndo());
     const canRedo = useWorkflowStore((state) => state.canRedo());
 
-    // Initialize workflow on mount
+    // Initialize workflow on mount. Waits for the spec catalog so defaults
+    // (allow_interrupt, prompt placeholders, etc.) come from one source.
     useEffect(() => {
+        if (specsLoading) return;
+
+        const startSpec = bySpecName.get(NodeType.START_CALL);
+        const fallbackStartNodes: FlowNode[] = startSpec
+            ? [buildNewNode(NodeType.START_CALL, { x: 200, y: 200 }, [], startSpec)]
+            : [];
+
         const initialNodes = initialFlow?.nodes?.length
-            ? initialFlow.nodes.map(node => ({
-                ...node,
-                data: {
-                    ...node.data,
-                    invalid: false,
-                    allow_interrupt: node.data.allow_interrupt !== undefined
-                        ? node.data.allow_interrupt
-                        : getDefaultAllowInterrupt(node.type),
-                }
-            }))
-            : defaultNodes;
+            ? initialFlow.nodes.map((node) => {
+                const fallbackAllowInterrupt = specAllowInterrupt(node.type, bySpecName) ?? false;
+                return {
+                    ...node,
+                    data: {
+                        ...node.data,
+                        invalid: false,
+                        allow_interrupt:
+                            node.data.allow_interrupt !== undefined
+                                ? node.data.allow_interrupt
+                                : fallbackAllowInterrupt,
+                    },
+                };
+            })
+            : fallbackStartNodes;
 
         initializeWorkflow(
             workflowId,
@@ -238,7 +167,7 @@ export const useWorkflowState = ({
             initialWorkflowConfigurations,
             initialWorkflowConfigurations?.dictionary ?? ''
         );
-    }, [workflowId, initialWorkflowName, initialFlow?.nodes, initialFlow?.edges, initialTemplateContextVariables, initialWorkflowConfigurations, initializeWorkflow]);
+    }, [workflowId, initialWorkflowName, initialFlow?.nodes, initialFlow?.edges, initialTemplateContextVariables, initialWorkflowConfigurations, initializeWorkflow, specsLoading, bySpecName]);
 
     // Set up keyboard shortcuts for undo/redo
     useEffect(() => {
@@ -280,8 +209,13 @@ export const useWorkflowState = ({
             y: window.innerHeight / 2,
         });
 
+        const spec = bySpecName.get(nodeType);
+        if (!spec) {
+            logger.warn({ nodeType }, "No spec registered for node type — cannot add");
+            return;
+        }
         const newNode = {
-            ...getNewNode(nodeType, position, nodes),
+            ...buildNewNode(nodeType, position, nodes, spec),
             selected: true, // Mark the new node as selected
         };
 
@@ -297,7 +231,7 @@ export const useWorkflowState = ({
             workflow_id: workflowId,
         });
         setIsAddNodePanelOpen(false);
-    }, [nodes, setIsAddNodePanelOpen, workflowId]);
+    }, [nodes, setIsAddNodePanelOpen, workflowId, bySpecName]);
 
     const handleNameChange = (e: React.ChangeEvent<HTMLInputElement>) => {
         setWorkflowName(e.target.value);