feat: add hybrid text + recording functionality in agents (#191)

* feat: add recording feature in agents * chore: pin pipecat version * feat: show usage in UI * chore: update pipecat
2026-07-22 11:51:04 +02:00 · 2026-03-16 15:04:08 +05:30 · 2026-03-16 15:04:08 +05:30 · 494c60d774
commit 494c60d774
parent f075bcb623
43 changed files with 2865 additions and 397 deletions
--- a/ui/src/components/flow/nodes/AgentNode.tsx
+++ b/ui/src/components/flow/nodes/AgentNode.tsx
@ -3,9 +3,10 @@ import { Edit, FileText, Headset, PlusIcon, Trash2Icon, Wrench } from "lucide-re
 import { memo, useCallback, useEffect, useMemo, useState } from "react";

 import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
-import type { DocumentResponseSchema, ToolResponse } from "@/client/types.gen";
+import type { DocumentResponseSchema, RecordingResponseSchema, ToolResponse } from "@/client/types.gen";
 import { DocumentBadges } from "@/components/flow/DocumentBadges";
 import { DocumentSelector } from "@/components/flow/DocumentSelector";
+import { MentionTextarea } from "@/components/flow/MentionTextarea";
 import { ToolBadges } from "@/components/flow/ToolBadges";
 import { ToolSelector } from "@/components/flow/ToolSelector";
 import { ExtractionVariable, FlowNodeData } from "@/components/flow/types";
@ -42,6 +43,7 @@ interface AgentNodeEditFormProps {
    setDocumentUuids: (value: string[]) => void;
    tools: ToolResponse[];
    documents: DocumentResponseSchema[];
+    recordings: RecordingResponseSchema[];
 }

 interface AgentNodeProps extends NodeProps {
@ -50,7 +52,7 @@ interface AgentNodeProps extends NodeProps {

 export const AgentNode = memo(({ data, selected, id }: AgentNodeProps) => {
    const { open, setOpen, handleSaveNodeData, handleDeleteNode } = useNodeHandlers({ id });
-    const { saveWorkflow, tools, documents } = useWorkflow();
+    const { saveWorkflow, tools, documents, recordings } = useWorkflow();

    // Form state
    const [prompt, setPrompt] = useState(data.prompt);
@ -229,6 +231,7 @@ export const AgentNode = memo(({ data, selected, id }: AgentNodeProps) => {
                        setDocumentUuids={setDocumentUuids}
                        tools={tools ?? []}
                        documents={documents ?? []}
+                        recordings={recordings ?? []}
                    />
                )}
            </NodeEditDialog>
@ -257,6 +260,7 @@ const AgentNodeEditForm = ({
    setDocumentUuids,
    tools,
    documents,
+    recordings,
 }: AgentNodeEditFormProps) => {
    const handleVariableNameChange = (idx: number, value: string) => {
        const newVars = [...variables];
@ -318,13 +322,12 @@ const AgentNodeEditForm = ({
                <Label className="text-xs text-muted-foreground">
                    Enter the prompt for the agent. This will be used to generate the agent&apos;s response. Prompt engineering&apos;s best practices apply.
                </Label>
-                <Textarea
+                <MentionTextarea
                    value={prompt}
-                    onChange={(e) => setPrompt(e.target.value)}
-                    className="min-h-[100px] max-h-[300px] resize-none"
-                    style={{
-                        overflowY: 'auto'
-                    }}
+                    onChange={setPrompt}
+                    className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
+                    placeholder="Enter a prompt"
+                    recordings={recordings}
                />
            </div>

--- a/ui/src/components/flow/nodes/EndCall.tsx
+++ b/ui/src/components/flow/nodes/EndCall.tsx
@ -3,6 +3,8 @@ import { Edit, OctagonX, PlusIcon, Trash2Icon } from "lucide-react";
 import { memo, useEffect, useMemo, useState } from "react";

 import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
+import type { RecordingResponseSchema } from "@/client/types.gen";
+import { MentionTextarea } from "@/components/flow/MentionTextarea";
 import { ExtractionVariable, FlowNodeData } from "@/components/flow/types";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
@ -29,6 +31,7 @@ interface EndCallEditFormProps {
    setVariables: (vars: ExtractionVariable[]) => void;
    addGlobalPrompt: boolean;
    setAddGlobalPrompt: (value: boolean) => void;
+    recordings: RecordingResponseSchema[];
 }

 interface EndCallNodeProps extends NodeProps {
@ -40,7 +43,7 @@ export const EndCall = memo(({ data, selected, id }: EndCallNodeProps) => {
        id,
        additionalData: { is_end: true }
    });
-    const { saveWorkflow } = useWorkflow();
+    const { saveWorkflow, recordings } = useWorkflow();

    // Form state
    const [prompt, setPrompt] = useState(data.prompt);
@ -157,6 +160,7 @@ export const EndCall = memo(({ data, selected, id }: EndCallNodeProps) => {
                        setVariables={setVariables}
                        addGlobalPrompt={addGlobalPrompt}
                        setAddGlobalPrompt={setAddGlobalPrompt}
+                        recordings={recordings ?? []}
                    />
                )}
            </NodeEditDialog>
@ -177,6 +181,7 @@ const EndCallEditForm = ({
    setVariables,
    addGlobalPrompt,
    setAddGlobalPrompt,
+    recordings,
 }: EndCallEditFormProps) => {
    const handleVariableNameChange = (idx: number, value: string) => {
        const newVars = [...variables];
@ -216,14 +221,12 @@ const EndCallEditForm = ({
            <Label className="text-xs text-muted-foreground">
                Enter the prompt for the agent. This will be used to generate the agent&apos;s response. Prompt engineering&apos;s best practices apply.
            </Label>
-            <Textarea
+            <MentionTextarea
                value={prompt}
-                onChange={(e) => setPrompt(e.target.value)}
-                className="min-h-[100px] max-h-[300px] resize-none"
-                style={{
-                    overflowY: 'auto'
-                }}
-                placeholder="Enter a dynamic prompt"
+                onChange={setPrompt}
+                className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
+                placeholder="Enter a prompt"
+                recordings={recordings}
            />
            <div className="flex items-center space-x-2">
                <Switch id="add-global-prompt" checked={addGlobalPrompt} onCheckedChange={setAddGlobalPrompt} />
--- a/ui/src/components/flow/nodes/GlobalNode.tsx
+++ b/ui/src/components/flow/nodes/GlobalNode.tsx
@ -3,11 +3,12 @@ import { Edit, Headset, Trash2Icon } from "lucide-react";
 import { memo, useEffect, useMemo, useState } from "react";

 import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
+import type { RecordingResponseSchema } from "@/client/types.gen";
+import { MentionTextarea } from "@/components/flow/MentionTextarea";
 import { FlowNodeData } from "@/components/flow/types";
 import { Button } from "@/components/ui/button";
 import { Input } from "@/components/ui/input";
 import { Label } from "@/components/ui/label";
-import { Textarea } from "@/components/ui/textarea";
 import { NODE_DOCUMENTATION_URLS } from "@/constants/documentation";

 import { NodeContent } from "./common/NodeContent";
@ -20,6 +21,7 @@ interface GlobalNodeEditFormProps {
    setPrompt: (value: string) => void;
    name: string;
    setName: (value: string) => void;
+    recordings: RecordingResponseSchema[];
 }

 interface GlobalNodeProps extends NodeProps {
@ -28,7 +30,7 @@ interface GlobalNodeProps extends NodeProps {

 export const GlobalNode = memo(({ data, selected, id }: GlobalNodeProps) => {
    const { open, setOpen, handleSaveNodeData, handleDeleteNode } = useNodeHandlers({ id });
-    const { saveWorkflow } = useWorkflow();
+    const { saveWorkflow, recordings } = useWorkflow();

    // Form state
    const [prompt, setPrompt] = useState(data.prompt);
@ -118,6 +120,7 @@ export const GlobalNode = memo(({ data, selected, id }: GlobalNodeProps) => {
                        setPrompt={setPrompt}
                        name={name}
                        setName={setName}
+                        recordings={recordings ?? []}
                    />
                )}
            </NodeEditDialog>
@ -129,7 +132,8 @@ const GlobalNodeEditForm = ({
    prompt,
    setPrompt,
    name,
-    setName
+    setName,
+    recordings,
 }: GlobalNodeEditFormProps) => {
    return (
        <div className="grid gap-2">
@ -146,13 +150,12 @@ const GlobalNodeEditForm = ({
            <Label className="text-xs text-muted-foreground">
                This is the global prompt. This will be added to the system prompt of all the agents.
            </Label>
-            <Textarea
+            <MentionTextarea
                value={prompt}
-                onChange={(e) => setPrompt(e.target.value)}
-                className="min-h-[100px] max-h-[300px] resize-none"
-                style={{
-                    overflowY: 'auto'
-                }}
+                onChange={setPrompt}
+                className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
+                placeholder="Enter a prompt"
+                recordings={recordings}
            />
        </div>
    );
--- a/ui/src/components/flow/nodes/StartCall.tsx
+++ b/ui/src/components/flow/nodes/StartCall.tsx
@ -4,8 +4,10 @@ import { memo, useCallback, useEffect, useMemo, useState } from "react";

 import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
 import type { DocumentResponseSchema, ToolResponse } from "@/client/types.gen";
+import type { RecordingResponseSchema } from "@/client/types.gen";
 import { DocumentBadges } from "@/components/flow/DocumentBadges";
 import { DocumentSelector } from "@/components/flow/DocumentSelector";
+import { MentionTextarea } from "@/components/flow/MentionTextarea";
 import { ToolBadges } from "@/components/flow/ToolBadges";
 import { ToolSelector } from "@/components/flow/ToolSelector";
 import { ExtractionVariable, FlowNodeData } from "@/components/flow/types";
@ -48,6 +50,7 @@ interface StartCallEditFormProps {
    setDocumentUuids: (value: string[]) => void;
    tools: ToolResponse[];
    documents: DocumentResponseSchema[];
+    recordings: RecordingResponseSchema[];
 }

 interface StartCallNodeProps extends NodeProps {
@ -59,7 +62,7 @@ export const StartCall = memo(({ data, selected, id }: StartCallNodeProps) => {
        id,
        additionalData: { is_start: true }
    });
-    const { saveWorkflow, tools, documents } = useWorkflow();
+    const { saveWorkflow, tools, documents, recordings } = useWorkflow();

    // Form state
    const [prompt, setPrompt] = useState(data.prompt ?? "");
@ -248,6 +251,7 @@ export const StartCall = memo(({ data, selected, id }: StartCallNodeProps) => {
                        setDocumentUuids={setDocumentUuids}
                        tools={tools ?? []}
                        documents={documents ?? []}
+                        recordings={recordings ?? []}
                    />
                )}
            </NodeEditDialog>
@ -282,6 +286,7 @@ const StartCallEditForm = ({
    setDocumentUuids,
    tools,
    documents,
+    recordings,
 }: StartCallEditFormProps) => {
    const handleVariableNameChange = (idx: number, value: string) => {
        const newVars = [...variables];
@ -325,14 +330,12 @@ const StartCallEditForm = ({
            <Label className="text-xs text-muted-foreground">
                Enter the prompt for the agent. This will be used to generate the agent&apos;s response. Prompt engineering&apos;s best practices apply.
            </Label>
-            <Textarea
+            <MentionTextarea
                value={prompt}
-                onChange={(e) => setPrompt(e.target.value)}
-                className="min-h-[100px] max-h-[300px] resize-none"
-                style={{
-                    overflowY: 'auto'
-                }}
+                onChange={setPrompt}
+                className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
                placeholder="Enter a prompt"
+                recordings={recordings}
            />
            <div className="flex items-center space-x-2">
                <Switch id="allow-interrupt" checked={allowInterrupt} onCheckedChange={setAllowInterrupt} />