From 27ad5d675accbde7cba6b92144c095fb46756d7c Mon Sep 17 00:00:00 2001 From: Ramnique Singh <30795890+ramnique@users.noreply.github.com> Date: Tue, 15 Jul 2025 23:34:28 +0530 Subject: [PATCH] bring back twilio integration --- apps/rowboat/app/actions/voice_actions.ts | 14 +- .../app/api/twilio/inbound_call/route.ts | 120 ++++++++++++++++++ .../app/api/twilio/turn/[callSid]/route.ts | 97 ++++++++++++++ apps/rowboat/app/api/twilio/utils.ts | 32 +++++ apps/rowboat/app/lib/mongodb.ts | 3 +- apps/rowboat/app/lib/types/voice_types.ts | 17 ++- .../app/projects/[projectId]/config/app.tsx | 59 +++++---- .../[projectId]/config/components/voice.tsx | 6 +- apps/rowboat/package-lock.json | 11 +- apps/rowboat/package.json | 2 +- 10 files changed, 320 insertions(+), 41 deletions(-) create mode 100644 apps/rowboat/app/api/twilio/inbound_call/route.ts create mode 100644 apps/rowboat/app/api/twilio/turn/[callSid]/route.ts create mode 100644 apps/rowboat/app/api/twilio/utils.ts diff --git a/apps/rowboat/app/actions/voice_actions.ts b/apps/rowboat/app/actions/voice_actions.ts index 52a3622f..de926435 100644 --- a/apps/rowboat/app/actions/voice_actions.ts +++ b/apps/rowboat/app/actions/voice_actions.ts @@ -5,6 +5,8 @@ import { twilioConfigsCollection } from "../lib/mongodb"; import { ObjectId } from "mongodb"; import twilio from 'twilio'; import { Twilio } from 'twilio'; +import { z } from "zod"; +import { WithStringId } from "../lib/types/types"; // Helper function to serialize MongoDB documents function serializeConfig(config: any) { @@ -16,7 +18,7 @@ function serializeConfig(config: any) { } // Real implementation for configuring Twilio number -export async function configureTwilioNumber(params: TwilioConfigParams): Promise { +export async function configureTwilioNumber(params: z.infer): Promise { console.log('configureTwilioNumber - Received params:', params); try { const client = twilio(params.account_sid, params.auth_token); @@ -56,7 +58,7 @@ export async function configureTwilioNumber(params: TwilioConfigParams): Promise } // Save Twilio configuration to MongoDB -export async function saveTwilioConfig(params: TwilioConfigParams): Promise { +async function saveTwilioConfig(params: z.infer): Promise> { console.log('saveTwilioConfig - Incoming params:', { ...params, label: { @@ -140,7 +142,7 @@ export async function saveTwilioConfig(params: TwilioConfigParams): Promise>[]> { console.log('getTwilioConfigs - Fetching for projectId:', projectId); const configs = await twilioConfigsCollection .find({ @@ -174,13 +176,13 @@ export async function deleteTwilioConfig(projectId: string, configId: string) { } // Mock implementation for testing/development -export async function mockConfigureTwilioNumber(params: TwilioConfigParams): Promise { +export async function mockConfigureTwilioNumber(params: z.infer): Promise { await new Promise(resolve => setTimeout(resolve, 1000)); await saveTwilioConfig(params); return { success: true }; } -export async function configureInboundCall( +async function configureInboundCall( phone_number: string, account_sid: string, auth_token: string, @@ -228,7 +230,7 @@ export async function configureInboundCall( throw new Error('Voice service must use a public URL, not localhost.'); } - const inboundUrl = `${baseUrl}/inbound?workflow_id=${workflow_id}`; + const inboundUrl = `${baseUrl}/api/twilio/inbound_call`; console.log('Setting up webhooks:', { voiceUrl: inboundUrl, statusCallback: `${baseUrl}/call-status`, diff --git a/apps/rowboat/app/api/twilio/inbound_call/route.ts b/apps/rowboat/app/api/twilio/inbound_call/route.ts new file mode 100644 index 00000000..bbae718d --- /dev/null +++ b/apps/rowboat/app/api/twilio/inbound_call/route.ts @@ -0,0 +1,120 @@ +import { getResponse } from "@/app/lib/agents"; +import { agentWorkflowsCollection, twilioConfigsCollection, twilioInboundCallsCollection } from "@/app/lib/mongodb"; +import { collectProjectTools } from "@/app/lib/project_tools"; +import { PrefixLogger } from "@/app/lib/utils"; +import VoiceResponse from "twilio/lib/twiml/VoiceResponse"; +import { ObjectId } from "mongodb"; +import { z } from "zod"; +import { TwilioInboundCall } from "@/app/lib/types/voice_types"; +import { hangup, reject, XmlResponse, ZStandardRequestParams } from "../utils"; + +export async function POST(request: Request) { + let logger = new PrefixLogger("twilioInboundCall"); + logger.log("Received inbound call request"); + const recvdAt = new Date(); + + /* + form data example + ... + { + Called: '+1571XXXXXXX', + ToState: 'VA', + CallerCountry: 'IN', + Direction: 'inbound', + CallerState: 'PXXXXXXX', + ToZip: '', + CallSid: 'CA...b0', + To: '+1571XXXXXXX', + CallerZip: '', + ToCountry: 'US', + StirVerstat: 'TN-Validation-Passed-C', + CallToken: '%7B...', + CalledZip: '', + ApiVersion: '2010-04-01', + CalledCity: '', + CallStatus: 'ringing', + From: '+919XXXXXXXXX', + AccountSid: 'A....1c', + CalledCountry: 'US', + CallerCity: '', + ToCity: '', + FromCountry: 'IN', + Caller: '+919XXXXXXXXX' + FromCity: '', + CalledState: 'VA', + FromZip: '', + FromState: 'PXXXXXXX' + } + */ + // parse and validate form data + const formData = await request.formData(); + logger.log('request body:', JSON.stringify(Object.fromEntries(formData))); + const data = ZStandardRequestParams.parse(Object.fromEntries(formData)); + logger = logger.child(data.To); + + // get a matching twilio config for this phone number. + // if not found, reject the call + const twilioConfig = await twilioConfigsCollection.findOne({ + phone_number: data.To, + status: 'active', + }); + if (!twilioConfig) { + logger.log('No active twilio config found for this phone number'); + return reject('rejected'); + } + + // extract workflow and project id and fetch workflow from db + // if workflow not found, reject the call + const projectId = twilioConfig.project_id; + const workflowId = twilioConfig.workflow_id; + const workflow = await agentWorkflowsCollection.findOne({ + projectId: projectId, + _id: new ObjectId(workflowId), + }); + if (!workflow) { + logger.log(`Workflow ${workflowId} not found for project ${projectId}`); + return reject('rejected'); + } + + // fetch project tools + const projectTools = await collectProjectTools(projectId); + + // this is the first turn, get the initial assistant response + // and validate it + const { messages } = await getResponse(workflow, projectTools, []); + if (messages.length === 0) { + logger.log('Agent response is empty'); + return hangup(); + } + const lastMessage = messages[messages.length - 1]; + if (lastMessage.role !== 'assistant' || !lastMessage.content) { + logger.log('Invalid last message'); + return hangup(); + } + + // save call state + const call: z.infer = { + callSid: data.CallSid, + to: data.To, + from: data.From, + projectId, + workflowId, + messages, + createdAt: recvdAt.toISOString(), + lastUpdatedAt: new Date().toISOString(), + }; + await twilioInboundCallsCollection.insertOne(call); + + // speak out response + const response = new VoiceResponse(); + response.say(lastMessage.content); + response.gather({ + input: ['speech'], + speechTimeout: 'auto', + language: 'en-US', + enhanced: true, + speechModel: 'phone_call', + action: `/api/twilio/turn/${data.CallSid}`, + }); + return XmlResponse(response); +} \ No newline at end of file diff --git a/apps/rowboat/app/api/twilio/turn/[callSid]/route.ts b/apps/rowboat/app/api/twilio/turn/[callSid]/route.ts new file mode 100644 index 00000000..a0832b4b --- /dev/null +++ b/apps/rowboat/app/api/twilio/turn/[callSid]/route.ts @@ -0,0 +1,97 @@ +import { getResponse } from "@/app/lib/agents"; +import { agentWorkflowsCollection, twilioConfigsCollection, twilioInboundCallsCollection } from "@/app/lib/mongodb"; +import { collectProjectTools } from "@/app/lib/project_tools"; +import { PrefixLogger } from "@/app/lib/utils"; +import VoiceResponse from "twilio/lib/twiml/VoiceResponse"; +import { ObjectId } from "mongodb"; +import { z } from "zod"; +import { hangup, XmlResponse, ZStandardRequestParams } from "../../utils"; +import { Message } from "@/app/lib/types/types"; + +const ZRequestData = ZStandardRequestParams.extend({ + SpeechResult: z.string(), + Confidence: z.string(), +}); + +export async function POST( + request: Request, + { params }: { params: Promise<{ callSid: string }> } +) { + const { callSid } = await params; + let logger = new PrefixLogger(`turn:${callSid}`); + logger.log("Received turn"); + + // parse and validate form data + const formData = await request.formData(); + logger.log('request body:', JSON.stringify(Object.fromEntries(formData))); + const data = ZRequestData.parse(Object.fromEntries(formData)); + + // get call state from db + // if not found, hangup the call + const call = await twilioInboundCallsCollection.findOne({ + callSid, + }); + if (!call) { + logger.log('Call not found'); + return hangup(); + } + const { workflowId, projectId } = call; + + // fetch workflow + const workflow = await agentWorkflowsCollection.findOne({ + projectId: projectId, + _id: new ObjectId(workflowId), + }); + if (!workflow) { + logger.log(`Workflow ${workflowId} not found for project ${projectId}`); + return hangup(); + } + + // fetch project tools + const projectTools = await collectProjectTools(projectId); + + // add user speech as user message, and get assistant response + const reqMessages: z.infer[] = [ + ...call.messages, + { + role: 'user', + content: data.SpeechResult, + } + ]; + const { messages } = await getResponse(workflow, projectTools, reqMessages); + if (messages.length === 0) { + logger.log('Agent response is empty'); + return hangup(); + } + const lastMessage = messages[messages.length - 1]; + if (lastMessage.role !== 'assistant' || !lastMessage.content) { + logger.log('Invalid last message'); + return hangup(); + } + + // save call state + await twilioInboundCallsCollection.updateOne({ + _id: call._id, + }, { + $set: { + messages: [ + ...reqMessages, + ...messages, + ], + lastUpdatedAt: new Date().toISOString(), + } + }); + + // speak out response + const response = new VoiceResponse(); + response.say(lastMessage.content); + response.gather({ + input: ['speech'], + speechTimeout: 'auto', + language: 'en-US', + enhanced: true, + speechModel: 'phone_call', + action: `/api/twilio/turn/${callSid}`, + }); + return XmlResponse(response); +} \ No newline at end of file diff --git a/apps/rowboat/app/api/twilio/utils.ts b/apps/rowboat/app/api/twilio/utils.ts new file mode 100644 index 00000000..4285ddbb --- /dev/null +++ b/apps/rowboat/app/api/twilio/utils.ts @@ -0,0 +1,32 @@ +import TwiML from "twilio/lib/twiml/TwiML"; +import VoiceResponse from "twilio/lib/twiml/VoiceResponse"; +import { z } from "zod"; + +export function XmlResponse(content: TwiML) { + return new Response(content.toString(), { + headers: { + "Content-Type": "text/xml", + }, + }); +} + +export function reject(reason: VoiceResponse.RejectAttributes['reason']) { + return XmlResponse(new VoiceResponse() + .reject({ + reason, + }) + ); +} + +export function hangup() { + return XmlResponse(new VoiceResponse() + .hangup() + ); +} + +export const ZStandardRequestParams = z.object({ + To: z.string(), + Direction: z.literal('inbound'), + CallSid: z.string(), + From: z.string(), +}); \ No newline at end of file diff --git a/apps/rowboat/app/lib/mongodb.ts b/apps/rowboat/app/lib/mongodb.ts index 564a3a44..f1ba5ddd 100644 --- a/apps/rowboat/app/lib/mongodb.ts +++ b/apps/rowboat/app/lib/mongodb.ts @@ -8,7 +8,7 @@ import { EmbeddingDoc } from "./types/datasource_types"; import { DataSourceDoc } from "./types/datasource_types"; import { DataSource } from "./types/datasource_types"; import { TestScenario, TestResult, TestRun, TestProfile, TestSimulation } from "./types/testing_types"; -import { TwilioConfig } from "./types/voice_types"; +import { TwilioConfig, TwilioInboundCall } from "./types/voice_types"; import { z } from 'zod'; import { apiV1 } from "rowboat-shared"; @@ -32,6 +32,7 @@ export const chatsCollection = db.collection>("chats" export const chatMessagesCollection = db.collection>("chat_messages"); export const twilioConfigsCollection = db.collection>("twilio_configs"); export const usersCollection = db.collection>("users"); +export const twilioInboundCallsCollection = db.collection>("twilio_inbound_calls"); // Create indexes twilioConfigsCollection.createIndexes([ diff --git a/apps/rowboat/app/lib/types/voice_types.ts b/apps/rowboat/app/lib/types/voice_types.ts index 7780103e..e7facebf 100644 --- a/apps/rowboat/app/lib/types/voice_types.ts +++ b/apps/rowboat/app/lib/types/voice_types.ts @@ -1,5 +1,6 @@ import { z } from 'zod'; import { WithId } from 'mongodb'; +import { Message } from './types'; export const TwilioConfigParams = z.object({ phone_number: z.string(), @@ -15,9 +16,6 @@ export const TwilioConfig = TwilioConfigParams.extend({ status: z.enum(['active', 'deleted']), }); -export type TwilioConfigParams = z.infer; -export type TwilioConfig = WithId>; - export interface TwilioConfigResponse { success: boolean; error?: string; @@ -29,4 +27,15 @@ export interface InboundConfigResponse { workflow_id: string; previous_webhook?: string; error?: string; -} \ No newline at end of file +} + +export const TwilioInboundCall = z.object({ + callSid: z.string(), + to: z.string(), + from: z.string(), + projectId: z.string(), + workflowId: z.string(), + messages: z.array(Message), + createdAt: z.string().datetime(), + lastUpdatedAt: z.string().datetime().optional(), +}) \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/config/app.tsx b/apps/rowboat/app/projects/[projectId]/config/app.tsx index 6e8b7c96..fe96316e 100644 --- a/apps/rowboat/app/projects/[projectId]/config/app.tsx +++ b/apps/rowboat/app/projects/[projectId]/config/app.tsx @@ -1,7 +1,7 @@ 'use client'; import { Metadata } from "next"; -import { Spinner, Textarea, Button, Dropdown, DropdownMenu, DropdownItem, DropdownTrigger, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Input, useDisclosure, Divider } from "@heroui/react"; +import { Spinner, Textarea, Button, Dropdown, DropdownMenu, DropdownItem, DropdownTrigger, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, Input, useDisclosure, Divider, Tab, Tabs } from "@heroui/react"; import { ReactNode, useEffect, useState } from "react"; import { getProjectConfig, updateProjectName, updateWebhookUrl, createApiKey, deleteApiKey, listApiKeys, deleteProject, rotateSecret } from "../../../actions/project_actions"; import { CopyButton } from "../../../../components/common/copy-button"; @@ -15,6 +15,7 @@ import { Label } from "../../../lib/components/label"; import { FormSection } from "../../../lib/components/form-section"; import { Panel } from "@/components/common/panel-common"; import { ProjectSection } from './components/project'; +import { VoiceSection } from "./components/voice"; export const metadata: Metadata = { title: "Project config", @@ -446,7 +447,7 @@ export function DeleteProjectSection({ const [projectName, setProjectName] = useState(""); const [projectNameInput, setProjectNameInput] = useState(""); const [confirmationInput, setConfirmationInput] = useState(""); - + const isValid = projectNameInput === projectName && confirmationInput === "delete project"; useEffect(() => { @@ -473,8 +474,8 @@ export function DeleteProjectSection({ This action cannot be undone.

- -
); } diff --git a/apps/rowboat/app/projects/[projectId]/config/components/voice.tsx b/apps/rowboat/app/projects/[projectId]/config/components/voice.tsx index 0c588f34..2c1c6ce2 100644 --- a/apps/rowboat/app/projects/[projectId]/config/components/voice.tsx +++ b/apps/rowboat/app/projects/[projectId]/config/components/voice.tsx @@ -9,6 +9,8 @@ import { TwilioConfig } from "../../../../lib/types/voice_types"; import { CheckCircleIcon, XCircleIcon, InfoIcon, EyeOffIcon, EyeIcon } from "lucide-react"; import { Section } from './project'; import { clsx } from 'clsx'; +import { WithStringId } from "../../../../lib/types/types"; +import { z } from 'zod'; function PhoneNumberSection({ value, @@ -149,7 +151,7 @@ export function VoiceSection({ projectId }: { projectId: string }) { authToken: '', label: '' }); - const [existingConfig, setExistingConfig] = useState(null); + const [existingConfig, setExistingConfig] = useState> | null>(null); const [error, setError] = useState(null); const [loading, setLoading] = useState(false); const [success, setSuccess] = useState(false); @@ -207,7 +209,7 @@ export function VoiceSection({ projectId }: { projectId: string }) { setError(null); const configParams = { - phone_number: formState.phone, + phone_number: formState.phone.replaceAll(/[^0-9\+]/g, ''), account_sid: formState.accountSid, auth_token: formState.authToken, label: formState.label, diff --git a/apps/rowboat/package-lock.json b/apps/rowboat/package-lock.json index fa08f6e3..c2f64785 100644 --- a/apps/rowboat/package-lock.json +++ b/apps/rowboat/package-lock.json @@ -62,7 +62,7 @@ "tailwind-merge": "^2.5.5", "tailwindcss-animate": "^1.0.7", "tiktoken": "^1.0.17", - "twilio": "^5.4.5", + "twilio": "^5.7.3", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.5" }, @@ -16115,11 +16115,12 @@ "license": "Unlicense" }, "node_modules/twilio": { - "version": "5.4.5", - "resolved": "https://registry.npmjs.org/twilio/-/twilio-5.4.5.tgz", - "integrity": "sha512-PIteif0CBOrA42SWZiT8IwUuqTNakAFgvXYWsrjEPGaDSczu/GvBs3vUock4S+UguXj7cV4qBswWgXs5ySjGNg==", + "version": "5.7.3", + "resolved": "https://registry.npmjs.org/twilio/-/twilio-5.7.3.tgz", + "integrity": "sha512-RuCjbQRLorFZrqd52KZ4JzeUbCbs/3KJVdawcAQ2yR53S2D0VwBQ+1Pkcnc20Y8QLKCP41TkQ98MHbF7upRhtA==", + "license": "MIT", "dependencies": { - "axios": "^1.7.8", + "axios": "^1.8.3", "dayjs": "^1.11.9", "https-proxy-agent": "^5.0.0", "jsonwebtoken": "^9.0.2", diff --git a/apps/rowboat/package.json b/apps/rowboat/package.json index 13506686..f37efe45 100644 --- a/apps/rowboat/package.json +++ b/apps/rowboat/package.json @@ -69,7 +69,7 @@ "tailwind-merge": "^2.5.5", "tailwindcss-animate": "^1.0.7", "tiktoken": "^1.0.17", - "twilio": "^5.4.5", + "twilio": "^5.7.3", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.5" },