feat: refactor node spec and add mcp tools (#244)

* refactor: carve out extraction panel

* refactor: create spec versions for node types

* refactor: create a GenericNode and remove custom nodes

* feat: add python and typescript sdk

* add dograh sdk

* fix: fetch draft workflow definition over published one

* fix: fix routes of SDKs to use code gen

* chore: remove doclink dependency to reduce image size

* chore: format files

* chore: bump pipecat

* feat: let mcp fetch archived workflows on demand

* chore: fix tests

* feat: add sdk documentation

* chore: change banner and add badge

api/routes/credentials.py

@@ -9,6 +9,7 @@ from pydantic import BaseModel
 from api.db import db_client
 from api.db.models import UserModel
 from api.enums import WebhookCredentialType
+from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 
 router = APIRouter(prefix="/credentials")
@@ -107,7 +108,13 @@ def build_credential_response(credential) -> CredentialResponse:
     )
 
 
-@router.get("/")
+@router.get(
+    "/",
+    **sdk_expose(
+        method="list_credentials",
+        description="List webhook credentials available to the authenticated organization.",
+    ),
+)
 async def list_credentials(
     user: UserModel = Depends(get_user),
 ) -> List[CredentialResponse]:

api/routes/knowledge_base.py

@@ -17,6 +17,7 @@ from api.schemas.knowledge_base import (
     DocumentUploadResponseSchema,
     ProcessDocumentRequestSchema,
 )
+from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 from api.services.posthog_client import capture_event
 from api.services.storage import storage_fs
@@ -135,6 +136,7 @@ async def process_document(
             document.id,
             request.s3_key,
             user.selected_organization_id,
+            str(user.provider_id),
             128,  # max_tokens (default)
             request.retrieval_mode,
         )
@@ -190,6 +192,10 @@ async def process_document(
     "/documents",
     response_model=DocumentListResponseSchema,
     summary="List documents",
+    **sdk_expose(
+        method="list_documents",
+        description="List knowledge base documents available to the authenticated organization.",
+    ),
 )
 async def list_documents(
     status: Annotated[

api/routes/main.py

@@ -8,6 +8,7 @@ from api.routes.credentials import router as credentials_router
 from api.routes.integration import router as integration_router
 from api.routes.knowledge_base import router as knowledge_base_router
 from api.routes.looptalk import router as looptalk_router
+from api.routes.node_types import router as node_types_router
 from api.routes.organization import router as organization_router
 from api.routes.organization_usage import router as organization_usage_router
 from api.routes.public_agent import router as public_agent_router
@@ -54,6 +55,7 @@ router.include_router(workflow_embed_router)
 router.include_router(knowledge_base_router)
 router.include_router(workflow_recording_router)
 router.include_router(auth_router)
+router.include_router(node_types_router)
 
 
 class HealthResponse(BaseModel):

api/routes/node_types.py

@@ -0,0 +1,67 @@
+"""API for the node-spec catalog.
+
+Exposes the registered NodeSpecs (one per node type) so frontend renderers
+and the LLM SDK can build forms / typed constructors from a single source
+of truth.
+
+Endpoints:
+    GET /node-types          → list every registered NodeSpec
+    GET /node-types/{name}   → single NodeSpec by name
+"""
+
+from fastapi import APIRouter, Depends, HTTPException
+from pydantic import BaseModel
+
+from api.db.models import UserModel
+from api.sdk_expose import sdk_expose
+from api.services.auth.depends import get_user
+from api.services.workflow.node_specs import (
+    SPEC_VERSION,
+    NodeSpec,
+    all_specs,
+    get_spec,
+)
+
+router = APIRouter(prefix="/node-types")
+
+
+class NodeTypesResponse(BaseModel):
+    spec_version: str
+    node_types: list[NodeSpec]
+
+
+@router.get(
+    "",
+    response_model=NodeTypesResponse,
+    **sdk_expose(
+        method="list_node_types",
+        description="List every registered node type with its spec. Pinned to spec_version.",
+    ),
+)
+async def list_node_types(
+    _user: UserModel = Depends(get_user),
+) -> NodeTypesResponse:
+    """List every registered NodeSpec.
+
+    SDK clients should pin to `spec_version` and warn if the server reports
+    a higher version than what they were generated against.
+    """
+    return NodeTypesResponse(spec_version=SPEC_VERSION, node_types=all_specs())
+
+
+@router.get(
+    "/{name}",
+    response_model=NodeSpec,
+    **sdk_expose(
+        method="get_node_type",
+        description="Fetch a single node spec by name.",
+    ),
+)
+async def get_node_type(
+    name: str,
+    _user: UserModel = Depends(get_user),
+) -> NodeSpec:
+    spec = get_spec(name)
+    if spec is None:
+        raise HTTPException(status_code=404, detail=f"Unknown node type: {name!r}")
+    return spec

api/routes/telephony.py

@@ -29,6 +29,7 @@ from api.db.workflow_client import WorkflowClient
 from api.db.workflow_run_client import WorkflowRunClient
 from api.enums import CallType, OrganizationConfigurationKey, WorkflowRunState
 from api.errors.telephony_errors import TelephonyError
+from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 from api.services.campaign.campaign_call_dispatcher import campaign_call_dispatcher
 from api.services.campaign.campaign_event_publisher import get_campaign_event_publisher
@@ -139,7 +140,13 @@ class StatusCallbackRequest(BaseModel):
         )
 
 
-@router.post("/initiate-call")
+@router.post(
+    "/initiate-call",
+    **sdk_expose(
+        method="test_phone_call",
+        description="Place a test call from a workflow to a phone number.",
+    ),
+)
 async def initiate_call(
     request: InitiateCallRequest, user: UserModel = Depends(get_user)
 ):

api/routes/tool.py

@@ -10,6 +10,7 @@ from pydantic import BaseModel, Field, field_validator
 from api.db import db_client
 from api.db.models import UserModel
 from api.enums import PostHogEvent, ToolCategory, ToolStatus
+from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 from api.services.posthog_client import capture_event
 
@@ -276,7 +277,13 @@ def validate_status(status: str) -> None:
             )
 
 
-@router.get("/")
+@router.get(
+    "/",
+    **sdk_expose(
+        method="list_tools",
+        description="List tools available to the authenticated organization.",
+    ),
+)
 async def list_tools(
     status: Optional[str] = None,
     category: Optional[str] = None,

api/routes/workflow.py

@@ -16,6 +16,7 @@ from api.db.models import UserModel
 from api.db.workflow_template_client import WorkflowTemplateClient
 from api.enums import CallType, PostHogEvent, StorageBackend
 from api.schemas.workflow import WorkflowRunResponseSchema
+from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 from api.services.campaign.report import generate_workflow_report_csv
 from api.services.configuration.check_validity import UserConfigurationValidator
@@ -27,7 +28,7 @@ from api.services.configuration.resolve import resolve_effective_config
 from api.services.mps_service_key_client import mps_service_key_client
 from api.services.posthog_client import capture_event
 from api.services.storage import storage_fs
-from api.services.workflow.dto import ReactFlowDTO
+from api.services.workflow.dto import ReactFlowDTO, sanitize_workflow_definition
 from api.services.workflow.duplicate import duplicate_workflow
 from api.services.workflow.errors import ItemKind, WorkflowError
 from api.services.workflow.workflow import WorkflowGraph
@@ -453,7 +454,13 @@ async def get_workflow_count(
     )
 
 
-@router.get("/fetch")
+@router.get(
+    "/fetch",
+    **sdk_expose(
+        method="list_workflows",
+        description="List all workflows in the authenticated organization.",
+    ),
+)
 async def get_workflows(
     user: UserModel = Depends(get_user),
     status: Optional[str] = Query(
@@ -499,7 +506,13 @@ async def get_workflows(
     ]
 
 
-@router.get("/fetch/{workflow_id}")
+@router.get(
+    "/fetch/{workflow_id}",
+    **sdk_expose(
+        method="get_workflow",
+        description="Get a single workflow by ID (returns draft if one exists, else published).",
+    ),
+)
 async def get_workflow(
     workflow_id: int,
     user: UserModel = Depends(get_user),
@@ -701,7 +714,13 @@ async def update_workflow_status(
         raise HTTPException(status_code=500, detail=str(e))
 
 
-@router.put("/{workflow_id}")
+@router.put(
+    "/{workflow_id}",
+    **sdk_expose(
+        method="update_workflow",
+        description="Update a workflow's name and/or definition. Saves as a new draft.",
+    ),
+)
 async def update_workflow(
     workflow_id: int,
     request: UpdateWorkflowRequest,
@@ -721,8 +740,10 @@ async def update_workflow(
         HTTPException: If the workflow is not found or if there's a database error
     """
     try:
-        # Restore real API keys where the incoming definition has masked placeholders
-        workflow_definition = request.workflow_definition
+        # Strip UI runtime-only fields (invalid, validationMessage, etc.) from
+        # node.data / edge.data before anything touches the DB — the UI sends
+        # nodes wholesale from the React Flow store, which carries those.
+        workflow_definition = sanitize_workflow_definition(request.workflow_definition)
         if workflow_definition:
             existing_workflow = await db_client.get_workflow(
                 workflow_id, organization_id=user.selected_organization_id

api/routes/workflow_recording.py

@@ -19,6 +19,7 @@ from api.schemas.workflow_recording import (
     RecordingUpdateRequestSchema,
     RecordingUploadResponseSchema,
 )
+from api.sdk_expose import sdk_expose
 from api.services.auth.depends import get_user
 from api.services.mps_service_key_client import mps_service_key_client
 from api.services.storage import storage_fs
@@ -165,6 +166,10 @@ async def create_recordings(
     "/",
     response_model=RecordingListResponseSchema,
     summary="List recordings",
+    **sdk_expose(
+        method="list_recordings",
+        description="List workflow recordings available to the authenticated organization.",
+    ),
 )
 async def list_recordings(
     workflow_id: Annotated[