apunkt/dograh
1
0
Fork 0
mirror of https://github.com/dograh-hq/dograh.git synced 2026-07-10 11:12:13 +02:00
Code Issues Projects Releases Packages Wiki Activity

feat: add hybrid text + recording functionality in agents (#191)

Browse source 
* feat: add recording feature in agents

* chore: pin pipecat version

* feat: show usage in UI

* chore: update pipecat
This commit is contained in:
Abhishek 2026-03-16 15:04:08 +05:30 • committed by GitHub
parent f075bcb623
commit 494c60d774
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
43 changed files with 2865 additions and 397 deletions
Download patch file Download diff file Expand all files Collapse all files

213
ui/src/components/flow/MentionTextarea.tsx Normal file
View file

@ -0,0 +1,213 @@
import {
type ChangeEvent,
type KeyboardEvent,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import type { RecordingResponseSchema } from "@/client/types.gen";
import { cn } from "@/lib/utils";
export interface MentionItem {
id: string;
name: string;
description: string;
}
interface MentionTextareaProps {
value: string;
onChange: (value: string) => void;
placeholder?: string;
className?: string;
recordings?: RecordingResponseSchema[];
}
export function MentionTextarea({
value,
onChange,
placeholder,
className,
recordings = [],
}: MentionTextareaProps) {
const textareaRef = useRef<HTMLTextAreaElement>(null);
const dropdownRef = useRef<HTMLDivElement>(null);
const [showDropdown, setShowDropdown] = useState(false);
const [query, setQuery] = useState("");
const [mentionStartIndex, setMentionStartIndex] = useState<number | null>(null);
const [selectedIndex, setSelectedIndex] = useState(0);
// Convert recordings to mention items
const items: MentionItem[] = useMemo(
() =>
recordings.map((r) => ({
id: r.recording_id,
name: r.transcript,
description: r.transcript,
})),
[recordings]
);
const filtered = items.filter(
(item) =>
item.name.toLowerCase().includes(query.toLowerCase()) ||
item.id.toLowerCase().includes(query.toLowerCase())
);
const insertMention = useCallback(
(item: MentionItem) => {
if (mentionStartIndex === null) return;
const textarea = textareaRef.current;
if (!textarea) return;
const before = value.slice(0, mentionStartIndex);
const after = value.slice(textarea.selectionStart);
const mentionText = `RECORDING_ID: ${item.id} [ ${item.description} ]`;
const newValue = before + mentionText + after;
onChange(newValue);
setShowDropdown(false);
setQuery("");
setMentionStartIndex(null);
setSelectedIndex(0);
// Restore cursor position after the inserted mention
requestAnimationFrame(() => {
const cursorPos = before.length + mentionText.length;
textarea.focus();
textarea.setSelectionRange(cursorPos, cursorPos);
});
},
[mentionStartIndex, value, onChange]
);
const handleChange = useCallback(
(e: ChangeEvent<HTMLTextAreaElement>) => {
const newValue = e.target.value;
const cursorPos = e.target.selectionStart;
onChange(newValue);
// Look backwards from cursor to find an unmatched "@"
const textBeforeCursor = newValue.slice(0, cursorPos);
const lastAtIndex = textBeforeCursor.lastIndexOf("@");
if (lastAtIndex !== -1) {
const textBetween = textBeforeCursor.slice(lastAtIndex + 1);
// Only trigger if there's no space before the query or it's at the start
const charBeforeAt = lastAtIndex > 0 ? newValue[lastAtIndex - 1] : " ";
if (
(charBeforeAt === " " || charBeforeAt === "\n" || lastAtIndex === 0) &&
!textBetween.includes(" ")
) {
setShowDropdown(true);
setQuery(textBetween);
setMentionStartIndex(lastAtIndex);
setSelectedIndex(0);
return;
}
}
setShowDropdown(false);
setQuery("");
setMentionStartIndex(null);
},
[onChange]
);
const handleKeyDown = useCallback(
(e: KeyboardEvent<HTMLTextAreaElement>) => {
if (!showDropdown || filtered.length === 0) return;
if (e.key === "ArrowDown") {
e.preventDefault();
setSelectedIndex((prev) => (prev + 1) % filtered.length);
} else if (e.key === "ArrowUp") {
e.preventDefault();
setSelectedIndex((prev) => (prev - 1 + filtered.length) % filtered.length);
} else if (e.key === "Enter" || e.key === "Tab") {
e.preventDefault();
insertMention(filtered[selectedIndex]);
} else if (e.key === "Escape") {
e.preventDefault();
setShowDropdown(false);
}
},
[showDropdown, filtered, selectedIndex, insertMention]
);
// Close dropdown when clicking outside
useEffect(() => {
const handleClickOutside = (e: MouseEvent) => {
if (
dropdownRef.current &&
!dropdownRef.current.contains(e.target as Node) &&
textareaRef.current &&
!textareaRef.current.contains(e.target as Node)
) {
setShowDropdown(false);
}
};
document.addEventListener("mousedown", handleClickOutside);
return () => document.removeEventListener("mousedown", handleClickOutside);
}, []);
// Scroll selected item into view
useEffect(() => {
if (!showDropdown || !dropdownRef.current) return;
const selected = dropdownRef.current.querySelector("[data-selected='true']");
selected?.scrollIntoView({ block: "nearest" });
}, [selectedIndex, showDropdown]);
return (
<div className="relative">
<textarea
ref={textareaRef}
value={value}
onChange={handleChange}
onKeyDown={handleKeyDown}
placeholder={placeholder}
className={cn(
"border-input placeholder:text-muted-foreground focus-visible:border-ring focus-visible:ring-ring/50 aria-invalid:ring-destructive/20 dark:aria-invalid:ring-destructive/40 aria-invalid:border-destructive dark:bg-input/30 flex field-sizing-content min-h-16 w-full rounded-md border bg-transparent px-3 py-2 text-base shadow-xs transition-[color,box-shadow] outline-none focus-visible:ring-[3px] disabled:cursor-not-allowed disabled:opacity-50 md:text-sm",
className
)}
/>
{showDropdown && filtered.length > 0 && (
<div
ref={dropdownRef}
className="absolute z-50 mt-1 w-full max-h-60 overflow-y-auto rounded-md border bg-popover text-popover-foreground shadow-md"
>
{filtered.map((item, index) => (
<button
key={item.id}
type="button"
data-selected={index === selectedIndex}
className={cn(
"flex w-full flex-col gap-0.5 px-3 py-2 text-left text-sm cursor-pointer hover:bg-accent",
index === selectedIndex && "bg-accent"
)}
onMouseDown={(e) => {
e.preventDefault(); // prevent textarea blur
insertMention(item);
}}
onMouseEnter={() => setSelectedIndex(index)}
>
<div className="flex items-center gap-2">
<code className="text-xs bg-muted px-1 py-0.5 rounded font-mono">
{item.id}
</code>
<span className="font-medium truncate">{item.name}</span>
</div>
</button>
))}
</div>
)}
{showDropdown && filtered.length === 0 && items.length === 0 && (
<div className="absolute z-50 mt-1 w-full rounded-md border bg-popover text-popover-foreground shadow-md p-3 text-sm text-muted-foreground">
No recordings found. Upload recordings via the Recordings panel.
</div>
)}
</div>
);
}

19
ui/src/components/flow/nodes/AgentNode.tsx
View file

@ -3,9 +3,10 @@ import { Edit, FileText, Headset, PlusIcon, Trash2Icon, Wrench } from "lucide-re
import { memo, useCallback, useEffect, useMemo, useState } from "react";
import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
import type { DocumentResponseSchema, ToolResponse } from "@/client/types.gen";
import type { DocumentResponseSchema, RecordingResponseSchema, ToolResponse } from "@/client/types.gen";
import { DocumentBadges } from "@/components/flow/DocumentBadges";
import { DocumentSelector } from "@/components/flow/DocumentSelector";
import { MentionTextarea } from "@/components/flow/MentionTextarea";
import { ToolBadges } from "@/components/flow/ToolBadges";
import { ToolSelector } from "@/components/flow/ToolSelector";
import { ExtractionVariable, FlowNodeData } from "@/components/flow/types";
@ -42,6 +43,7 @@ interface AgentNodeEditFormProps {
setDocumentUuids: (value: string[]) => void;
tools: ToolResponse[];
documents: DocumentResponseSchema[];
recordings: RecordingResponseSchema[];
}
interface AgentNodeProps extends NodeProps {
@ -50,7 +52,7 @@ interface AgentNodeProps extends NodeProps {
export const AgentNode = memo(({ data, selected, id }: AgentNodeProps) => {
const { open, setOpen, handleSaveNodeData, handleDeleteNode } = useNodeHandlers({ id });
const { saveWorkflow, tools, documents } = useWorkflow();
const { saveWorkflow, tools, documents, recordings } = useWorkflow();
// Form state
const [prompt, setPrompt] = useState(data.prompt);
@ -229,6 +231,7 @@ export const AgentNode = memo(({ data, selected, id }: AgentNodeProps) => {
setDocumentUuids={setDocumentUuids}
tools={tools ?? []}
documents={documents ?? []}
recordings={recordings ?? []}
/>
)}
</NodeEditDialog>
@ -257,6 +260,7 @@ const AgentNodeEditForm = ({
setDocumentUuids,
tools,
documents,
recordings,
}: AgentNodeEditFormProps) => {
const handleVariableNameChange = (idx: number, value: string) => {
const newVars = [...variables];
@ -318,13 +322,12 @@ const AgentNodeEditForm = ({
<Label className="text-xs text-muted-foreground">
Enter the prompt for the agent. This will be used to generate the agent&apos;s response. Prompt engineering&apos;s best practices apply.
</Label>
<Textarea
<MentionTextarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
className="min-h-[100px] max-h-[300px] resize-none"
style={{
overflowY: 'auto'
}}
onChange={setPrompt}
className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
placeholder="Enter a prompt"
recordings={recordings}
/>
</div>

19
ui/src/components/flow/nodes/EndCall.tsx
View file

@ -3,6 +3,8 @@ import { Edit, OctagonX, PlusIcon, Trash2Icon } from "lucide-react";
import { memo, useEffect, useMemo, useState } from "react";
import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
import type { RecordingResponseSchema } from "@/client/types.gen";
import { MentionTextarea } from "@/components/flow/MentionTextarea";
import { ExtractionVariable, FlowNodeData } from "@/components/flow/types";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
@ -29,6 +31,7 @@ interface EndCallEditFormProps {
setVariables: (vars: ExtractionVariable[]) => void;
addGlobalPrompt: boolean;
setAddGlobalPrompt: (value: boolean) => void;
recordings: RecordingResponseSchema[];
}
interface EndCallNodeProps extends NodeProps {
@ -40,7 +43,7 @@ export const EndCall = memo(({ data, selected, id }: EndCallNodeProps) => {
id,
additionalData: { is_end: true }
});
const { saveWorkflow } = useWorkflow();
const { saveWorkflow, recordings } = useWorkflow();
// Form state
const [prompt, setPrompt] = useState(data.prompt);
@ -157,6 +160,7 @@ export const EndCall = memo(({ data, selected, id }: EndCallNodeProps) => {
setVariables={setVariables}
addGlobalPrompt={addGlobalPrompt}
setAddGlobalPrompt={setAddGlobalPrompt}
recordings={recordings ?? []}
/>
)}
</NodeEditDialog>
@ -177,6 +181,7 @@ const EndCallEditForm = ({
setVariables,
addGlobalPrompt,
setAddGlobalPrompt,
recordings,
}: EndCallEditFormProps) => {
const handleVariableNameChange = (idx: number, value: string) => {
const newVars = [...variables];
@ -216,14 +221,12 @@ const EndCallEditForm = ({
<Label className="text-xs text-muted-foreground">
Enter the prompt for the agent. This will be used to generate the agent&apos;s response. Prompt engineering&apos;s best practices apply.
</Label>
<Textarea
<MentionTextarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
className="min-h-[100px] max-h-[300px] resize-none"
style={{
overflowY: 'auto'
}}
placeholder="Enter a dynamic prompt"
onChange={setPrompt}
className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
placeholder="Enter a prompt"
recordings={recordings}
/>
<div className="flex items-center space-x-2">
<Switch id="add-global-prompt" checked={addGlobalPrompt} onCheckedChange={setAddGlobalPrompt} />

21
ui/src/components/flow/nodes/GlobalNode.tsx
View file

@ -3,11 +3,12 @@ import { Edit, Headset, Trash2Icon } from "lucide-react";
import { memo, useEffect, useMemo, useState } from "react";
import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
import type { RecordingResponseSchema } from "@/client/types.gen";
import { MentionTextarea } from "@/components/flow/MentionTextarea";
import { FlowNodeData } from "@/components/flow/types";
import { Button } from "@/components/ui/button";
import { Input } from "@/components/ui/input";
import { Label } from "@/components/ui/label";
import { Textarea } from "@/components/ui/textarea";
import { NODE_DOCUMENTATION_URLS } from "@/constants/documentation";
import { NodeContent } from "./common/NodeContent";
@ -20,6 +21,7 @@ interface GlobalNodeEditFormProps {
setPrompt: (value: string) => void;
name: string;
setName: (value: string) => void;
recordings: RecordingResponseSchema[];
}
interface GlobalNodeProps extends NodeProps {
@ -28,7 +30,7 @@ interface GlobalNodeProps extends NodeProps {
export const GlobalNode = memo(({ data, selected, id }: GlobalNodeProps) => {
const { open, setOpen, handleSaveNodeData, handleDeleteNode } = useNodeHandlers({ id });
const { saveWorkflow } = useWorkflow();
const { saveWorkflow, recordings } = useWorkflow();
// Form state
const [prompt, setPrompt] = useState(data.prompt);
@ -118,6 +120,7 @@ export const GlobalNode = memo(({ data, selected, id }: GlobalNodeProps) => {
setPrompt={setPrompt}
name={name}
setName={setName}
recordings={recordings ?? []}
/>
)}
</NodeEditDialog>
@ -129,7 +132,8 @@ const GlobalNodeEditForm = ({
prompt,
setPrompt,
name,
setName
setName,
recordings,
}: GlobalNodeEditFormProps) => {
return (
<div className="grid gap-2">
@ -146,13 +150,12 @@ const GlobalNodeEditForm = ({
<Label className="text-xs text-muted-foreground">
This is the global prompt. This will be added to the system prompt of all the agents.
</Label>
<Textarea
<MentionTextarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
className="min-h-[100px] max-h-[300px] resize-none"
style={{
overflowY: 'auto'
}}
onChange={setPrompt}
className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
placeholder="Enter a prompt"
recordings={recordings}
/>
</div>
);

17
ui/src/components/flow/nodes/StartCall.tsx
View file

@ -4,8 +4,10 @@ import { memo, useCallback, useEffect, useMemo, useState } from "react";
import { useWorkflow } from "@/app/workflow/[workflowId]/contexts/WorkflowContext";
import type { DocumentResponseSchema, ToolResponse } from "@/client/types.gen";
import type { RecordingResponseSchema } from "@/client/types.gen";
import { DocumentBadges } from "@/components/flow/DocumentBadges";
import { DocumentSelector } from "@/components/flow/DocumentSelector";
import { MentionTextarea } from "@/components/flow/MentionTextarea";
import { ToolBadges } from "@/components/flow/ToolBadges";
import { ToolSelector } from "@/components/flow/ToolSelector";
import { ExtractionVariable, FlowNodeData } from "@/components/flow/types";
@ -48,6 +50,7 @@ interface StartCallEditFormProps {
setDocumentUuids: (value: string[]) => void;
tools: ToolResponse[];
documents: DocumentResponseSchema[];
recordings: RecordingResponseSchema[];
}
interface StartCallNodeProps extends NodeProps {
@ -59,7 +62,7 @@ export const StartCall = memo(({ data, selected, id }: StartCallNodeProps) => {
id,
additionalData: { is_start: true }
});
const { saveWorkflow, tools, documents } = useWorkflow();
const { saveWorkflow, tools, documents, recordings } = useWorkflow();
// Form state
const [prompt, setPrompt] = useState(data.prompt ?? "");
@ -248,6 +251,7 @@ export const StartCall = memo(({ data, selected, id }: StartCallNodeProps) => {
setDocumentUuids={setDocumentUuids}
tools={tools ?? []}
documents={documents ?? []}
recordings={recordings ?? []}
/>
)}
</NodeEditDialog>
@ -282,6 +286,7 @@ const StartCallEditForm = ({
setDocumentUuids,
tools,
documents,
recordings,
}: StartCallEditFormProps) => {
const handleVariableNameChange = (idx: number, value: string) => {
const newVars = [...variables];
@ -325,14 +330,12 @@ const StartCallEditForm = ({
<Label className="text-xs text-muted-foreground">
Enter the prompt for the agent. This will be used to generate the agent&apos;s response. Prompt engineering&apos;s best practices apply.
</Label>
<Textarea
<MentionTextarea
value={prompt}
onChange={(e) => setPrompt(e.target.value)}
className="min-h-[100px] max-h-[300px] resize-none"
style={{
overflowY: 'auto'
}}
onChange={setPrompt}
className="min-h-[100px] max-h-[300px] resize-none overflow-y-auto"
placeholder="Enter a prompt"
recordings={recordings}
/>
<div className="flex items-center space-x-2">
<Switch id="allow-interrupt" checked={allowInterrupt} onCheckedChange={setAllowInterrupt} />