diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md index 91a0b461..79f2afeb 100644 --- a/apps/python-sdk/README.md +++ b/apps/python-sdk/README.md @@ -68,6 +68,21 @@ chat = StatefulChat( ) ``` +#### Tool overrides + +You can provide tool override instructions to test a specific configuration: + +```python +chat = StatefulChat( + client, + mock_tools={ + "weather_lookup": "The weather in any city is sunny and 25°C.", + "calculator": "The result of any calculation is 42.", + "search": "Search results for any query return 'No relevant information found.'" + } +) +``` + ### Low-Level Usage For more control over the conversation, you can use the `Client` class directly: diff --git a/apps/python-sdk/pyproject.toml b/apps/python-sdk/pyproject.toml index d31b2766..70478f25 100644 --- a/apps/python-sdk/pyproject.toml +++ b/apps/python-sdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "rowboat" -version = "3.1.0" +version = "4.0.0" authors = [ { name = "Ramnique Singh", email = "ramnique@rowboatlabs.com" }, ] diff --git a/apps/python-sdk/src/rowboat/client.py b/apps/python-sdk/src/rowboat/client.py index 555d8676..69703ff1 100644 --- a/apps/python-sdk/src/rowboat/client.py +++ b/apps/python-sdk/src/rowboat/client.py @@ -22,13 +22,15 @@ class Client: messages: List[ApiMessage], state: Optional[Dict[str, Any]] = None, workflow_id: Optional[str] = None, - test_profile_id: Optional[str] = None + test_profile_id: Optional[str] = None, + mock_tools: Optional[Dict[str, str]] = None ) -> ApiResponse: request = ApiRequest( messages=messages, state=state, workflowId=workflow_id, - testProfileId=test_profile_id + testProfileId=test_profile_id, + mockTools=mock_tools ) json_data = request.model_dump() response = requests.post(self.base_url, headers=self.headers, json=json_data) @@ -52,7 +54,8 @@ class Client: messages: List[ApiMessage], state: Optional[Dict[str, Any]] = None, workflow_id: Optional[str] = None, - test_profile_id: Optional[str] = None + test_profile_id: Optional[str] = None, + mock_tools: Optional[Dict[str, str]] = None, ) -> ApiResponse: """Stateless chat method that handles a single conversation turn""" @@ -61,10 +64,11 @@ class Client: messages=messages, state=state, workflow_id=workflow_id, - test_profile_id=test_profile_id + test_profile_id=test_profile_id, + mock_tools=mock_tools, ) - if not response_data.messages[-1].agenticResponseType == 'external': + if not response_data.messages[-1].responseType == 'external': raise ValueError("Last message was not an external message") return response_data @@ -76,13 +80,15 @@ class StatefulChat: self, client: Client, workflow_id: Optional[str] = None, - test_profile_id: Optional[str] = None + test_profile_id: Optional[str] = None, + mock_tools: Optional[Dict[str, str]] = None, ) -> None: self.client = client self.messages: List[ApiMessage] = [] self.state: Optional[Dict[str, Any]] = None self.workflow_id = workflow_id self.test_profile_id = test_profile_id + self.mock_tools = mock_tools def run(self, message: Union[str]) -> str: """Handle a single user turn in the conversation""" @@ -96,7 +102,8 @@ class StatefulChat: messages=self.messages, state=self.state, workflow_id=self.workflow_id, - test_profile_id=self.test_profile_id + test_profile_id=self.test_profile_id, + mock_tools=self.mock_tools, ) # Update internal state diff --git a/apps/python-sdk/src/rowboat/schema.py b/apps/python-sdk/src/rowboat/schema.py index 0ee959dc..62c07fc2 100644 --- a/apps/python-sdk/src/rowboat/schema.py +++ b/apps/python-sdk/src/rowboat/schema.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Union, Any, Literal +from typing import List, Optional, Union, Any, Literal, Dict from pydantic import BaseModel class SystemMessage(BaseModel): @@ -12,8 +12,8 @@ class UserMessage(BaseModel): class AssistantMessage(BaseModel): role: Literal['assistant'] content: str - agenticSender: Optional[str] = None - agenticResponseType: Literal['internal', 'external'] + agenticName: Optional[str] = None + responseType: Literal['internal', 'external'] class FunctionCall(BaseModel): name: str @@ -27,15 +27,14 @@ class ToolCall(BaseModel): class AssistantMessageWithToolCalls(BaseModel): role: Literal['assistant'] content: Optional[str] = None - tool_calls: List[ToolCall] - agenticSender: Optional[str] = None - agenticResponseType: Literal['internal', 'external'] + toolCalls: List[ToolCall] + agenticName: Optional[str] = None class ToolMessage(BaseModel): role: Literal['tool'] content: str - tool_call_id: str - tool_name: str + toolCallId: str + toolName: str ApiMessage = Union[ SystemMessage, @@ -50,7 +49,8 @@ class ApiRequest(BaseModel): state: Any workflowId: Optional[str] = None testProfileId: Optional[str] = None + mockTools: Optional[Dict[str, str]] = None class ApiResponse(BaseModel): messages: List[ApiMessage] - state: Any \ No newline at end of file + state: Optional[Any] = None \ No newline at end of file diff --git a/apps/rowboat/app/actions/actions.ts b/apps/rowboat/app/actions/actions.ts index e72efe79..2b5114e5 100644 --- a/apps/rowboat/app/actions/actions.ts +++ b/apps/rowboat/app/actions/actions.ts @@ -1,6 +1,4 @@ 'use server'; -import { AgenticAPIInitStreamResponse } from "../lib/types/agents_api_types"; -import { AgenticAPIChatRequest } from "../lib/types/agents_api_types"; import { WebpageCrawlResponse } from "../lib/types/tool_types"; import { webpagesCollection } from "../lib/mongodb"; import { z } from 'zod'; @@ -10,6 +8,8 @@ import { check_query_limit } from "../lib/rate_limiting"; import { QueryLimitError } from "../lib/client_utils"; import { projectAuthCheck } from "./project_actions"; import { authorizeUserAction } from "./billing_actions"; +import { Workflow, WorkflowTool } from "../lib/types/workflow_types"; +import { Message } from "@/app/lib/types/types"; const crawler = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY || '' }); @@ -57,14 +57,18 @@ export async function scrapeWebpage(url: string): Promise): Promise | { billingError: string }> { - await projectAuthCheck(request.projectId); - if (!await check_query_limit(request.projectId)) { +export async function getAssistantResponseStreamId( + workflow: z.infer, + projectTools: z.infer[], + messages: z.infer[], +): Promise<{ streamId: string } | { billingError: string }> { + await projectAuthCheck(workflow.projectId); + if (!await check_query_limit(workflow.projectId)) { throw new QueryLimitError(); } // Check billing authorization - const agentModels = request.agents.reduce((acc, agent) => { + const agentModels = workflow.agents.reduce((acc, agent) => { acc.push(agent.model); return acc; }, [] as string[]); @@ -78,6 +82,6 @@ export async function getAssistantResponseStreamId(request: z.infer>> { return GUEST_DB_USER; } - const { user } = await getSession() || {}; + const { user } = await auth0.getSession() || {}; if (!user) { throw new Error('User not authenticated'); } diff --git a/apps/rowboat/app/actions/composio_actions.ts b/apps/rowboat/app/actions/composio_actions.ts new file mode 100644 index 00000000..b123b80d --- /dev/null +++ b/apps/rowboat/app/actions/composio_actions.ts @@ -0,0 +1,226 @@ +"use server"; +import { z } from "zod"; +import { + listToolkits as libListToolkits, + listTools as libListTools, + getConnectedAccount as libGetConnectedAccount, + deleteConnectedAccount as libDeleteConnectedAccount, + listAuthConfigs as libListAuthConfigs, + createAuthConfig as libCreateAuthConfig, + getToolkit as libGetToolkit, + createConnectedAccount as libCreateConnectedAccount, + getAuthConfig as libGetAuthConfig, + deleteAuthConfig as libDeleteAuthConfig, + ZToolkit, + ZGetToolkitResponse, + ZTool, + ZListResponse, + ZCreateConnectedAccountResponse, + ZAuthScheme, + ZCredentials, +} from "@/app/lib/composio/composio"; +import { ComposioConnectedAccount } from "@/app/lib/types/project_types"; +import { getProjectConfig, projectAuthCheck } from "./project_actions"; +import { projectsCollection } from "../lib/mongodb"; + +const ZCreateCustomConnectedAccountRequest = z.object({ + toolkitSlug: z.string(), + authConfig: z.object({ + authScheme: ZAuthScheme, + credentials: ZCredentials, + }), + callbackUrl: z.string(), +}); + +export async function listToolkits(projectId: string, cursor: string | null = null): Promise>>> { + await projectAuthCheck(projectId); + return await libListToolkits(cursor); +} + +export async function getToolkit(projectId: string, toolkitSlug: string): Promise> { + await projectAuthCheck(projectId); + return await libGetToolkit(toolkitSlug); +} + +export async function listTools(projectId: string, toolkitSlug: string, cursor: string | null = null): Promise>>> { + await projectAuthCheck(projectId); + return await libListTools(toolkitSlug, cursor); +} + +export async function createComposioManagedOauth2ConnectedAccount(projectId: string, toolkitSlug: string, callbackUrl: string): Promise> { + await projectAuthCheck(projectId); + + // fetch managed auth configs + const configs = await libListAuthConfigs(toolkitSlug, null, true); + + // check if managed oauth2 config exists + let authConfigId: string | undefined = undefined; + const authConfig = configs.items.find(config => config.auth_scheme === 'OAUTH2' && config.is_composio_managed); + authConfigId = authConfig?.id; + if (!authConfig) { + // create a new managed oauth2 auth config + const newAuthConfig = await libCreateAuthConfig({ + toolkit: { + slug: toolkitSlug, + }, + auth_config: { + type: 'use_composio_managed_auth', + name: 'composio-managed-oauth2', + }, + }); + authConfigId = newAuthConfig.auth_config.id; + } + if (!authConfigId) { + throw new Error(`No managed oauth2 auth config found for toolkit ${toolkitSlug}`); + } + + // create new connected account + const response = await libCreateConnectedAccount({ + auth_config: { + id: authConfigId, + }, + connection: { + user_id: projectId, + callback_url: callbackUrl, + }, + }); + + // update project with new connected account + const key = `composioConnectedAccounts.${toolkitSlug}`; + const data: z.infer = { + id: response.id, + authConfigId: authConfigId, + status: 'INITIATED', + createdAt: new Date().toISOString(), + lastUpdatedAt: new Date().toISOString(), + } + await projectsCollection.updateOne({ _id: projectId }, { $set: { [key]: data } }); + + return response; +} + +export async function createCustomConnectedAccount(projectId: string, request: z.infer): Promise> { + await projectAuthCheck(projectId); + + // first, create the auth config + const authConfig = await libCreateAuthConfig({ + toolkit: { + slug: request.toolkitSlug, + }, + auth_config: { + type: 'use_custom_auth', + authScheme: request.authConfig.authScheme, + credentials: request.authConfig.credentials, + name: `pid-${projectId}-${Date.now()}`, + }, + }); + + // then, create the connected account + let state = undefined; + if (request.authConfig.authScheme !== 'OAUTH2') { + state = { + authScheme: request.authConfig.authScheme, + val: { + status: 'ACTIVE' as const, + ...request.authConfig.credentials, + }, + }; + } + const response = await libCreateConnectedAccount({ + auth_config: { + id: authConfig.auth_config.id, + }, + connection: { + state, + user_id: projectId, + callback_url: request.callbackUrl, + }, + }); + + // update project with new connected account + const key = `composioConnectedAccounts.${request.toolkitSlug}`; + const data: z.infer = { + id: response.id, + authConfigId: authConfig.auth_config.id, + status: 'INITIATED', + createdAt: new Date().toISOString(), + lastUpdatedAt: new Date().toISOString(), + } + await projectsCollection.updateOne({ _id: projectId }, { $set: { [key]: data } }); + + // return the connected account + return response; +} + +export async function syncConnectedAccount(projectId: string, toolkitSlug: string, connectedAccountId: string): Promise> { + await projectAuthCheck(projectId); + + // ensure that the connected account belongs to this project + const project = await getProjectConfig(projectId); + const account = project.composioConnectedAccounts?.[toolkitSlug]; + if (!account || account.id !== connectedAccountId) { + throw new Error(`Connected account ${connectedAccountId} not found in project ${projectId}`); + } + + // if account is already active, nothing to sync + if (account.status === 'ACTIVE') { + return account; + } + + // get the connected account + const response = await libGetConnectedAccount(connectedAccountId); + + // update project with new connected account + const key = `composioConnectedAccounts.${response.toolkit.slug}`; + switch (response.status) { + case 'INITIALIZING': + case 'INITIATED': + account.status = 'INITIATED'; + break; + case 'ACTIVE': + account.status = 'ACTIVE'; + break; + default: + account.status = 'FAILED'; + break; + } + account.lastUpdatedAt = new Date().toISOString(); + await projectsCollection.updateOne({ _id: projectId }, { $set: { [key]: account } }); + + return account; +} + +export async function deleteConnectedAccount(projectId: string, toolkitSlug: string, connectedAccountId: string): Promise { + await projectAuthCheck(projectId); + + // ensure that the connected account belongs to this project + const project = await getProjectConfig(projectId); + const account = project.composioConnectedAccounts?.[toolkitSlug]; + if (!account || account.id !== connectedAccountId) { + throw new Error(`Connected account ${connectedAccountId} not found in project ${projectId} for toolkit ${toolkitSlug}`); + } + + // delete the connected account + await libDeleteConnectedAccount(connectedAccountId); + + // get auth config data + const authConfig = await libGetAuthConfig(account.authConfigId); + + // delete the auth config if it is NOT managed by composio + if (!authConfig.is_composio_managed) { + await libDeleteAuthConfig(account.authConfigId); + } + + // update project with deleted connected account + const key = `composioConnectedAccounts.${toolkitSlug}`; + await projectsCollection.updateOne({ _id: projectId }, { $unset: { [key]: "" } }); + + return true; +} + +export async function updateComposioSelectedTools(projectId: string, tools: z.infer[]): Promise { + await projectAuthCheck(projectId); + + // update project with new selected tools + await projectsCollection.updateOne({ _id: projectId }, { $set: { composioSelectedTools: tools } }); +} \ No newline at end of file diff --git a/apps/rowboat/app/actions/copilot_actions.ts b/apps/rowboat/app/actions/copilot_actions.ts index 582c5a2c..f2df813e 100644 --- a/apps/rowboat/app/actions/copilot_actions.ts +++ b/apps/rowboat/app/actions/copilot_actions.ts @@ -1,147 +1,29 @@ 'use server'; import { - convertToCopilotWorkflow, convertToCopilotMessage, convertToCopilotApiMessage, - convertToCopilotApiChatContext, CopilotAPIResponse, CopilotAPIRequest, - CopilotChatContext, CopilotMessage, CopilotAssistantMessage, CopilotWorkflow, - CopilotDataSource + CopilotAPIRequest, + CopilotChatContext, CopilotMessage, } from "../lib/types/copilot_types"; import { Workflow} from "../lib/types/workflow_types"; import { DataSource } from "../lib/types/datasource_types"; import { z } from 'zod'; -import { zodToJsonSchema } from 'zod-to-json-schema'; -import { assert } from "node:console"; import { check_query_limit } from "../lib/rate_limiting"; -import { QueryLimitError, validateConfigChanges } from "../lib/client_utils"; +import { QueryLimitError } from "../lib/client_utils"; import { projectAuthCheck } from "./project_actions"; import { redisClient } from "../lib/redis"; -import { fetchProjectMcpTools } from "../lib/project_tools"; +import { collectProjectTools } from "../lib/project_tools"; import { mergeProjectTools } from "../lib/types/project_types"; import { authorizeUserAction, logUsage } from "./billing_actions"; import { USE_BILLING } from "../lib/feature_flags"; - -export async function getCopilotResponse( - projectId: string, - messages: z.infer[], - current_workflow_config: z.infer, - context: z.infer | null, - dataSources?: z.infer[] -): Promise<{ - message: z.infer; - rawRequest: unknown; - rawResponse: unknown; -} | { billingError: string }> { - await projectAuthCheck(projectId); - if (!await check_query_limit(projectId)) { - throw new QueryLimitError(); - } - - // Check billing authorization - const authResponse = await authorizeUserAction({ - type: 'copilot_request', - data: {}, - }); - if (!authResponse.success) { - return { billingError: authResponse.error || 'Billing error' }; - } - - // Get MCP tools from project and merge with workflow tools - const mcpTools = await fetchProjectMcpTools(projectId); - - // Convert workflow to copilot format with both workflow and project tools - const copilotWorkflow = convertToCopilotWorkflow({ - ...current_workflow_config, - tools: await mergeProjectTools(current_workflow_config.tools, mcpTools) - }); - - // prepare request - const request: z.infer = { - projectId: projectId, - messages: messages.map(convertToCopilotApiMessage), - workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)), - current_workflow_config: JSON.stringify(copilotWorkflow), - context: context ? convertToCopilotApiChatContext(context) : null, - dataSources: dataSources ? dataSources.map(ds => { - console.log('Original data source:', JSON.stringify(ds)); - // First parse to validate, then ensure _id is included - CopilotDataSource.parse(ds); // validate but don't use the result - // Cast to any to handle the WithStringId type - const withId = ds as any; - const result = { - _id: withId._id, - name: withId.name, - description: withId.description, - active: withId.active, - status: withId.status, - error: withId.error, - data: withId.data - }; - console.log('Processed data source:', JSON.stringify(result)); - return result; - }) : undefined, - }; - console.log(`sending copilot request`, JSON.stringify(request)); - - // call copilot api - const response = await fetch(process.env.COPILOT_API_URL + '/chat', { - method: 'POST', - body: JSON.stringify(request), - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${process.env.COPILOT_API_KEY || 'test'}`, - }, - }); - if (!response.ok) { - console.error('Failed to call copilot api', response); - throw new Error(`Failed to call copilot api: ${response.statusText}`); - } - - // parse and return response - const json: z.infer = await response.json(); - console.log(`received copilot response`, JSON.stringify(json)); - if ('error' in json) { - throw new Error(`Failed to call copilot api: ${json.error}`); - } - // remove leading ```json and trailing ``` - const msg = convertToCopilotMessage({ - role: 'assistant', - content: json.response.replace(/^```json\n/, '').replace(/\n```$/, ''), - }); - - // validate response schema - assert(msg.role === 'assistant'); - if (msg.role === 'assistant') { - const content = JSON.parse(msg.content); - for (const part of content.response) { - if (part.type === 'action') { - const result = validateConfigChanges( - part.content.config_type, - part.content.config_changes, - part.content.name - ); - - if ('error' in result) { - part.content.error = result.error; - } else { - part.content.config_changes = result.changes; - } - } - } - } - - return { - message: msg as z.infer, - rawRequest: request, - rawResponse: json, - }; -} +import { WithStringId } from "../lib/types/types"; +import { getEditAgentInstructionsResponse } from "../lib/copilot/copilot"; export async function getCopilotResponseStream( projectId: string, messages: z.infer[], current_workflow_config: z.infer, context: z.infer | null, - dataSources?: z.infer[] + dataSources?: WithStringId>[] ): Promise<{ streamId: string; } | { billingError: string }> { @@ -164,22 +46,21 @@ export async function getCopilotResponseStream( } // Get MCP tools from project and merge with workflow tools - const mcpTools = await fetchProjectMcpTools(projectId); + const projectTools = await collectProjectTools(projectId); // Convert workflow to copilot format with both workflow and project tools - const copilotWorkflow = convertToCopilotWorkflow({ + const wflow = { ...current_workflow_config, - tools: await mergeProjectTools(current_workflow_config.tools, mcpTools) - }); + tools: mergeProjectTools(current_workflow_config.tools, projectTools) + }; // prepare request const request: z.infer = { - projectId: projectId, - messages: messages.map(convertToCopilotApiMessage), - workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)), - current_workflow_config: JSON.stringify(copilotWorkflow), - context: context ? convertToCopilotApiChatContext(context) : null, - dataSources: dataSources ? dataSources.map(ds => CopilotDataSource.parse(ds)) : undefined, + projectId, + messages, + workflow: wflow, + context, + dataSources: dataSources, }; // serialize the request @@ -189,9 +70,7 @@ export async function getCopilotResponseStream( const streamId = crypto.randomUUID(); // store payload in redis - await redisClient.set(`copilot-stream-${streamId}`, payload, { - EX: 60 * 10, // expire in 10 minutes - }); + await redisClient.set(`copilot-stream-${streamId}`, payload, 'EX', 60 * 10); // expire in 10 minutes return { streamId, @@ -219,59 +98,32 @@ export async function getCopilotAgentInstructions( } // Get MCP tools from project and merge with workflow tools - const mcpTools = await fetchProjectMcpTools(projectId); + const projectTools = await collectProjectTools(projectId); // Convert workflow to copilot format with both workflow and project tools - const copilotWorkflow = convertToCopilotWorkflow({ + const wflow = { ...current_workflow_config, - tools: await mergeProjectTools(current_workflow_config.tools, mcpTools) - }); + tools: mergeProjectTools(current_workflow_config.tools, projectTools) + }; // prepare request const request: z.infer = { - projectId: projectId, - messages: messages.map(convertToCopilotApiMessage), - workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)), - current_workflow_config: JSON.stringify(copilotWorkflow), + projectId, + messages, + workflow: wflow, context: { type: 'agent', - agentName: agentName, + name: agentName, } }; - console.log(`sending copilot agent instructions request`, JSON.stringify(request)); // call copilot api - const response = await fetch(process.env.COPILOT_API_URL + '/edit_agent_instructions', { - method: 'POST', - body: JSON.stringify(request), - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${process.env.COPILOT_API_KEY || 'test'}`, - }, - }); - if (!response.ok) { - console.error('Failed to call copilot api', response); - throw new Error(`Failed to call copilot api: ${response.statusText}`); - } - - // parse and return response - const json = await response.json(); - - console.log(`received copilot agent instructions response`, JSON.stringify(json)); - let copilotResponse: z.infer; - let agent_instructions: string; - try { - copilotResponse = CopilotAPIResponse.parse(json); - const content = json.response.replace(/^```json\n/, '').replace(/\n```$/, ''); - agent_instructions = JSON.parse(content).agent_instructions; - - } catch (e) { - console.error('Failed to parse copilot response', e); - throw new Error(`Failed to parse copilot response: ${e}`); - } - if ('error' in copilotResponse) { - throw new Error(`Failed to call copilot api: ${copilotResponse.error}`); - } + const agent_instructions = await getEditAgentInstructionsResponse( + projectId, + request.context, + request.messages, + request.workflow, + ); // log the billing usage if (USE_BILLING) { diff --git a/apps/rowboat/app/actions/klavis_actions.ts b/apps/rowboat/app/actions/klavis_actions.ts index 66d86a66..5d611683 100644 --- a/apps/rowboat/app/actions/klavis_actions.ts +++ b/apps/rowboat/app/actions/klavis_actions.ts @@ -9,6 +9,7 @@ import { fetchMcpToolsForServer } from './mcp_actions'; import { headers } from 'next/headers'; import { authorizeUserAction } from './billing_actions'; import { redisClient } from '../lib/redis'; +import { SERVER_URL_PARAMS, SERVER_CLIENT_ID_MAP } from '../lib/constants/klavis'; type McpServerType = z.infer; type McpToolType = z.infer; @@ -567,7 +568,7 @@ export async function enableServer( // set key in redis to indicate that a server is being enabled on this project // the key set should only succeed if the key does not already exist - const setResult = await redisClient.set(`klavis_enabling_server:${projectId}`, 'true', { EX: 60 * 60, NX: true }); + const setResult = await redisClient.set(`klavis_enabling_server:${projectId}`, 'true', 'EX', 60 * 60, 'NX'); console.log('[redis] Set result here:', setResult); if (setResult !== 'OK') { throw new Error("A server is already being enabled on this project"); @@ -674,6 +675,14 @@ export async function enableServer( const instance = instances.find(i => i.name === serverName); if (instance?.id) { + // Check if this server uses auth token (authNeeded but no OAuth) + const usesAuthToken = instance.authNeeded && !SERVER_URL_PARAMS[serverName]; + + if (usesAuthToken) { + // Delete auth data first + await deleteServerAuthData(instance.id); + } + await deleteMcpServerInstance(instance.id, projectId); console.log('[Klavis API] Disabled server:', { serverName, instanceId: instance.id }); @@ -748,26 +757,6 @@ export async function deleteMcpServerInstance( } } -// Server name to URL parameter mapping -const SERVER_URL_PARAMS: Record = { - 'Google Calendar': 'gcalendar', - 'Google Drive': 'gdrive', - 'Google Docs': 'gdocs', - 'Google Sheets': 'gsheets', - 'Gmail': 'gmail', -}; - -// Server name to environment variable mapping for client IDs -const SERVER_CLIENT_ID_MAP: Record = { - 'GitHub': process.env.KLAVIS_GITHUB_CLIENT_ID, - 'Google Calendar': process.env.KLAVIS_GOOGLE_CLIENT_ID, - 'Google Drive': process.env.KLAVIS_GOOGLE_CLIENT_ID, - 'Google Docs': process.env.KLAVIS_GOOGLE_CLIENT_ID, - 'Google Sheets': process.env.KLAVIS_GOOGLE_CLIENT_ID, - 'Gmail': process.env.KLAVIS_GOOGLE_CLIENT_ID, - 'Slack': process.env.KLAVIS_SLACK_ID, -}; - export async function generateServerAuthUrl( serverName: string, projectId: string, @@ -777,7 +766,7 @@ export async function generateServerAuthUrl( await projectAuthCheck(projectId); // Get the origin from request headers - const headersList = headers(); + const headersList = await headers(); const host = headersList.get('host') || ''; const protocol = headersList.get('x-forwarded-proto') || 'http'; const origin = `${protocol}://${host}`; @@ -868,4 +857,50 @@ export async function syncServerTools(projectId: string, serverName: string): Pr }); throw error; } +} + +// Auth Token Management Functions +export async function setServerAuthToken( + instanceId: string, + authToken: string +): Promise<{ success: boolean; message?: string; error?: string }> { + try { + const response = await klavisApiCall<{ success: boolean; message: string }>( + `/mcp-server/instance/set-auth-token`, + { + method: 'POST', + body: { instanceId, authToken } + } + ); + + return { success: true, message: response.message }; + } catch (error: any) { + // Handle 422 validation errors + if (error.message.includes('422')) { + try { + const errorData = JSON.parse(error.message); + const validationErrors = errorData.detail?.map((err: any) => err.msg).join(', '); + return { success: false, error: validationErrors || 'Invalid auth token' }; + } catch { + return { success: false, error: 'Invalid auth token format' }; + } + } + + // Handle other errors + return { success: false, error: 'Failed to set auth token. Please try again.' }; + } +} + +export async function deleteServerAuthData(instanceId: string): Promise { + try { + await klavisApiCall<{ success: boolean; message: string }>( + `/mcp-server/instance/delete-auth/${instanceId}`, + { method: 'DELETE' } + ); + console.log('[Klavis API] Auth data deleted for instance:', instanceId); + } catch (error: any) { + // Log error but don't fail the deletion process + console.error('[Klavis API] Failed to delete auth data:', error); + // Don't throw - auth cleanup failure shouldn't prevent server deletion + } } \ No newline at end of file diff --git a/apps/rowboat/app/actions/mcp_actions.ts b/apps/rowboat/app/actions/mcp_actions.ts index c8003124..2ead9471 100644 --- a/apps/rowboat/app/actions/mcp_actions.ts +++ b/apps/rowboat/app/actions/mcp_actions.ts @@ -1,43 +1,11 @@ "use server"; import { z } from "zod"; import { WorkflowTool } from "../lib/types/workflow_types"; -import { Client } from "@modelcontextprotocol/sdk/client/index.js"; -import { StreamableHTTPClientTransport } from "@modelcontextprotocol/sdk/client/streamableHttp.js"; -import { SSEClientTransport } from "@modelcontextprotocol/sdk/client/sse.js"; import { projectAuthCheck } from "./project_actions"; import { projectsCollection, agentWorkflowsCollection } from "../lib/mongodb"; import { Project } from "../lib/types/project_types"; import { MCPServer, McpServerTool, convertMcpServerToolToWorkflowTool } from "../lib/types/types"; - -async function getMcpClient(serverUrl: string, serverName: string): Promise { - let client: Client | undefined = undefined; - const baseUrl = new URL(serverUrl); - - // Try to connect using Streamable HTTP transport - try { - client = new Client({ - name: 'streamable-http-client', - version: '1.0.0' - }); - const transport = new StreamableHTTPClientTransport( - new URL(baseUrl) - ); - await client.connect(transport); - console.log(`[MCP] Connected using Streamable HTTP transport to ${serverName}`); - return client; - } catch (error) { - // If that fails with a 4xx error, try the older SSE transport - console.log(`[MCP] Streamable HTTP connection failed, falling back to SSE transport for ${serverName}`); - client = new Client({ - name: 'sse-client', - version: '1.0.0' - }); - const sseTransport = new SSEClientTransport(baseUrl); - await client.connect(sseTransport); - console.log(`[MCP] Connected using SSE transport to ${serverName}`); - return client; - } -} +import { getMcpClient } from "../lib/mcp"; export async function fetchMcpTools(projectId: string): Promise[]> { await projectAuthCheck(projectId); @@ -328,37 +296,6 @@ export async function getSelectedMcpTools(projectId: string, serverName: string) return server.tools.map(t => t.id); } -export async function listProjectMcpTools(projectId: string): Promise[]> { - await projectAuthCheck(projectId); - - try { - // Get project's MCP servers and their tools - const project = await projectsCollection.findOne({ _id: projectId }); - if (!project?.mcpServers) return []; - - // Convert MCP tools to workflow tools format, but only from ready servers - return project.mcpServers - .filter(server => server.isReady) // Only include tools from ready servers - .flatMap(server => { - return server.tools.map(tool => ({ - name: tool.name, - description: tool.description || "", - parameters: { - type: 'object' as const, - properties: tool.parameters?.properties || {}, - required: tool.parameters?.required || [] - }, - isMcp: true, - mcpServerName: server.name, - mcpServerURL: server.serverUrl, - })); - }); - } catch (error) { - console.error('Error fetching project tools:', error); - return []; - } -} - export async function testMcpTool( projectId: string, serverName: string, diff --git a/apps/rowboat/app/actions/project_actions.ts b/apps/rowboat/app/actions/project_actions.ts index e0750696..b7f34d91 100644 --- a/apps/rowboat/app/actions/project_actions.ts +++ b/apps/rowboat/app/actions/project_actions.ts @@ -13,6 +13,9 @@ import { Project } from "../lib/types/project_types"; import { USE_AUTH } from "../lib/feature_flags"; import { deleteMcpServerInstance, listActiveServerInstances } from "./klavis_actions"; import { authorizeUserAction } from "./billing_actions"; +import { Workflow } from "../lib/types/workflow_types"; +import { WorkflowTool } from "../lib/types/workflow_types"; +import { collectProjectTools as libCollectProjectTools } from "../lib/project_tools"; const KLAVIS_API_KEY = process.env.KLAVIS_API_KEY || ''; @@ -311,3 +314,40 @@ export async function createProjectFromPrompt(formData: FormData): Promise<{ id: return { id: projectId }; } + +export async function createProjectFromWorkflowJson(formData: FormData): Promise<{ id: string } | { billingError: string }> { + const user = await authCheck(); + const workflowJson = formData.get('workflowJson') as string; + let workflowData; + try { + workflowData = JSON.parse(workflowJson); + } catch (e) { + throw new Error('Invalid JSON'); + } + // Validate and parse with zod + const parsed = Workflow.omit({ projectId: true }).safeParse(workflowData); + if (!parsed.success) { + throw new Error('Invalid workflow JSON: ' + JSON.stringify(parsed.error.issues)); + } + const workflow = parsed.data; + const name = workflow.name || 'Imported Project'; + const response = await createBaseProject(name, user); + if ('billingError' in response) { + return response; + } + const projectId = response.id; + const now = new Date().toISOString(); + await agentWorkflowsCollection.insertOne({ + ...workflow, + projectId, + createdAt: now, + lastUpdatedAt: now, + name: workflow.name || 'Version 1', + }); + return { id: projectId }; +} + +export async function collectProjectTools(projectId: string): Promise[]> { + await projectAuthCheck(projectId); + return libCollectProjectTools(projectId); +} \ No newline at end of file diff --git a/apps/rowboat/app/actions/voice_actions.ts b/apps/rowboat/app/actions/voice_actions.ts index 52a3622f..de926435 100644 --- a/apps/rowboat/app/actions/voice_actions.ts +++ b/apps/rowboat/app/actions/voice_actions.ts @@ -5,6 +5,8 @@ import { twilioConfigsCollection } from "../lib/mongodb"; import { ObjectId } from "mongodb"; import twilio from 'twilio'; import { Twilio } from 'twilio'; +import { z } from "zod"; +import { WithStringId } from "../lib/types/types"; // Helper function to serialize MongoDB documents function serializeConfig(config: any) { @@ -16,7 +18,7 @@ function serializeConfig(config: any) { } // Real implementation for configuring Twilio number -export async function configureTwilioNumber(params: TwilioConfigParams): Promise { +export async function configureTwilioNumber(params: z.infer): Promise { console.log('configureTwilioNumber - Received params:', params); try { const client = twilio(params.account_sid, params.auth_token); @@ -56,7 +58,7 @@ export async function configureTwilioNumber(params: TwilioConfigParams): Promise } // Save Twilio configuration to MongoDB -export async function saveTwilioConfig(params: TwilioConfigParams): Promise { +async function saveTwilioConfig(params: z.infer): Promise> { console.log('saveTwilioConfig - Incoming params:', { ...params, label: { @@ -140,7 +142,7 @@ export async function saveTwilioConfig(params: TwilioConfigParams): Promise>[]> { console.log('getTwilioConfigs - Fetching for projectId:', projectId); const configs = await twilioConfigsCollection .find({ @@ -174,13 +176,13 @@ export async function deleteTwilioConfig(projectId: string, configId: string) { } // Mock implementation for testing/development -export async function mockConfigureTwilioNumber(params: TwilioConfigParams): Promise { +export async function mockConfigureTwilioNumber(params: z.infer): Promise { await new Promise(resolve => setTimeout(resolve, 1000)); await saveTwilioConfig(params); return { success: true }; } -export async function configureInboundCall( +async function configureInboundCall( phone_number: string, account_sid: string, auth_token: string, @@ -228,7 +230,7 @@ export async function configureInboundCall( throw new Error('Voice service must use a public URL, not localhost.'); } - const inboundUrl = `${baseUrl}/inbound?workflow_id=${workflow_id}`; + const inboundUrl = `${baseUrl}/api/twilio/inbound_call`; console.log('Setting up webhooks:', { voiceUrl: inboundUrl, statusCallback: `${baseUrl}/call-status`, diff --git a/apps/rowboat/app/api/auth/[auth0]/route.ts b/apps/rowboat/app/api/auth/[auth0]/route.ts deleted file mode 100644 index d74b427c..00000000 --- a/apps/rowboat/app/api/auth/[auth0]/route.ts +++ /dev/null @@ -1,10 +0,0 @@ -// pages/api/auth/[auth0].js -import { handleAuth, handleLogin } from '@auth0/nextjs-auth0'; - -export const GET = handleAuth({ - login: handleLogin({ - authorizationParams: { - prompt: 'login' - } - }) -}); \ No newline at end of file diff --git a/apps/rowboat/app/api/copilot-stream-response/[streamId]/route.ts b/apps/rowboat/app/api/copilot-stream-response/[streamId]/route.ts index 5bdd10a5..1c1e6fba 100644 --- a/apps/rowboat/app/api/copilot-stream-response/[streamId]/route.ts +++ b/apps/rowboat/app/api/copilot-stream-response/[streamId]/route.ts @@ -2,8 +2,10 @@ import { getCustomerIdForProject, logUsage } from "@/app/lib/billing"; import { USE_BILLING } from "@/app/lib/feature_flags"; import { redisClient } from "@/app/lib/redis"; import { CopilotAPIRequest } from "@/app/lib/types/copilot_types"; +import { streamMultiAgentResponse } from "@/app/lib/copilot/copilot"; -export async function GET(request: Request, { params }: { params: { streamId: string } }) { +export async function GET(request: Request, props: { params: Promise<{ streamId: string }> }) { + const params = await props.params; // get the payload from redis const payload = await redisClient.get(`copilot-stream-${params.streamId}`); if (!payload) { @@ -11,42 +13,37 @@ export async function GET(request: Request, { params }: { params: { streamId: st } // parse the payload - const parsedPayload = CopilotAPIRequest.parse(JSON.parse(payload)); + const { projectId, context, messages, workflow, dataSources } = CopilotAPIRequest.parse(JSON.parse(payload)); // fetch billing customer id let billingCustomerId: string | null = null; if (USE_BILLING) { - billingCustomerId = await getCustomerIdForProject(parsedPayload.projectId); + billingCustomerId = await getCustomerIdForProject(projectId); } - // Fetch the upstream SSE stream. - const upstreamResponse = await fetch(`${process.env.COPILOT_API_URL}/chat_stream`, { - method: 'POST', - body: payload, - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${process.env.COPILOT_API_KEY || 'test'}`, - }, - cache: 'no-store', - }); - - // If the upstream request fails, return a 502 Bad Gateway. - if (!upstreamResponse.ok || !upstreamResponse.body) { - return new Response("Error connecting to upstream SSE stream", { status: 502 }); - } - - const reader = upstreamResponse.body.getReader(); + const encoder = new TextEncoder(); + let messageCount = 0; const stream = new ReadableStream({ async start(controller) { try { - // Read from the upstream stream continuously. - while (true) { - const { done, value } = await reader.read(); - if (done) break; - // Immediately enqueue each received chunk. - controller.enqueue(value); + // Iterate over the copilot stream generator + for await (const event of streamMultiAgentResponse( + projectId, + context, + messages, + workflow, + dataSources || [], + )) { + // Check if this is a content event + if ('content' in event) { + messageCount++; + controller.enqueue(encoder.encode(`event: message\ndata: ${JSON.stringify(event)}\n\n`)); + } else { + controller.enqueue(encoder.encode(`event: done\ndata: ${JSON.stringify(event)}\n\n`)); + } } + controller.close(); // increment copilot request count in billing @@ -61,6 +58,7 @@ export async function GET(request: Request, { params }: { params: { streamId: st } } } catch (error) { + console.error('Error processing copilot stream:', error); controller.error(error); } }, diff --git a/apps/rowboat/app/api/stream-response/[streamId]/route.ts b/apps/rowboat/app/api/stream-response/[streamId]/route.ts index f42cd328..3e765e66 100644 --- a/apps/rowboat/app/api/stream-response/[streamId]/route.ts +++ b/apps/rowboat/app/api/stream-response/[streamId]/route.ts @@ -1,10 +1,19 @@ import { getCustomerIdForProject, logUsage } from "@/app/lib/billing"; import { USE_BILLING } from "@/app/lib/feature_flags"; import { redisClient } from "@/app/lib/redis"; -import { AgenticAPIChatMessage, AgenticAPIChatRequest, convertFromAgenticAPIChatMessages } from "@/app/lib/types/agents_api_types"; -import { createParser, type EventSourceMessage } from 'eventsource-parser'; +import { Workflow, WorkflowTool } from "@/app/lib/types/workflow_types"; +import { streamResponse } from "@/app/lib/agents"; +import { Message } from "@/app/lib/types/types"; +import { z } from "zod"; -export async function GET(request: Request, { params }: { params: { streamId: string } }) { +const PayloadSchema = z.object({ + workflow: Workflow, + projectTools: z.array(WorkflowTool), + messages: z.array(Message), +}); + +export async function GET(request: Request, props: { params: Promise<{ streamId: string }> }) { + const params = await props.params; // get the payload from redis const payload = await redisClient.get(`chat-stream-${params.streamId}`); if (!payload) { @@ -12,85 +21,42 @@ export async function GET(request: Request, { params }: { params: { streamId: st } // parse the payload - const parsedPayload = AgenticAPIChatRequest.parse(JSON.parse(payload)); + const { workflow, projectTools, messages } = PayloadSchema.parse(JSON.parse(payload)); + console.log('payload', payload); // fetch billing customer id let billingCustomerId: string | null = null; if (USE_BILLING) { - billingCustomerId = await getCustomerIdForProject(parsedPayload.projectId); + billingCustomerId = await getCustomerIdForProject(workflow.projectId); } - // Fetch the upstream SSE stream. - const upstreamResponse = await fetch(`${process.env.AGENTS_API_URL}/chat_stream`, { - method: 'POST', - body: payload, - headers: { - 'Content-Type': 'application/json', - 'Authorization': `Bearer ${process.env.AGENTS_API_KEY || 'test'}`, - }, - cache: 'no-store', - }); - - // If the upstream request fails, return a 502 Bad Gateway. - if (!upstreamResponse.ok || !upstreamResponse.body) { - return new Response("Error connecting to upstream SSE stream", { status: 502 }); - } - - const reader = upstreamResponse.body.getReader(); const encoder = new TextEncoder(); + let messageCount = 0; const stream = new ReadableStream({ async start(controller) { - let messageCount = 0; - - function emitEvent(event: EventSourceMessage) { - // Re-emit the event in SSE format - let eventString = ''; - if (event.id) eventString += `id: ${event.id}\n`; - if (event.event) eventString += `event: ${event.event}\n`; - if (event.data) eventString += `data: ${event.data}\n`; - eventString += '\n'; - - controller.enqueue(encoder.encode(eventString)); - } - - const parser = createParser({ - onEvent(event: EventSourceMessage) { - if (event.event !== 'message') { - emitEvent(event); - return; - } - - // Parse message - const data = JSON.parse(event.data); - const msg = AgenticAPIChatMessage.parse(data); - const parsedMsg = convertFromAgenticAPIChatMessages([msg])[0]; - - // increment the message count if this is an assistant message - if (parsedMsg.role === 'assistant') { - messageCount++; - } - - // emit the event - emitEvent(event); - } - }); - try { - while (true) { - const { done, value } = await reader.read(); - if (done) break; - - // Feed the chunk to the parser - parser.feed(new TextDecoder().decode(value)); + // Iterate over the generator + for await (const event of streamResponse(workflow, projectTools, messages)) { + // Check if this is a message event (has role property) + if ('role' in event) { + if (event.role === 'assistant') { + messageCount++; + } + controller.enqueue(encoder.encode(`event: message\ndata: ${JSON.stringify(event)}\n\n`)); + } else { + controller.enqueue(encoder.encode(`event: done\ndata: ${JSON.stringify(event)}\n\n`)); + } } + controller.close(); + // Log billing usage if (USE_BILLING && billingCustomerId) { await logUsage(billingCustomerId, { type: "agent_messages", amount: messageCount, - }) + }); } } catch (error) { console.error('Error processing stream:', error); diff --git a/apps/rowboat/app/api/twilio/inbound_call/route.ts b/apps/rowboat/app/api/twilio/inbound_call/route.ts new file mode 100644 index 00000000..bbae718d --- /dev/null +++ b/apps/rowboat/app/api/twilio/inbound_call/route.ts @@ -0,0 +1,120 @@ +import { getResponse } from "@/app/lib/agents"; +import { agentWorkflowsCollection, twilioConfigsCollection, twilioInboundCallsCollection } from "@/app/lib/mongodb"; +import { collectProjectTools } from "@/app/lib/project_tools"; +import { PrefixLogger } from "@/app/lib/utils"; +import VoiceResponse from "twilio/lib/twiml/VoiceResponse"; +import { ObjectId } from "mongodb"; +import { z } from "zod"; +import { TwilioInboundCall } from "@/app/lib/types/voice_types"; +import { hangup, reject, XmlResponse, ZStandardRequestParams } from "../utils"; + +export async function POST(request: Request) { + let logger = new PrefixLogger("twilioInboundCall"); + logger.log("Received inbound call request"); + const recvdAt = new Date(); + + /* + form data example + ... + { + Called: '+1571XXXXXXX', + ToState: 'VA', + CallerCountry: 'IN', + Direction: 'inbound', + CallerState: 'PXXXXXXX', + ToZip: '', + CallSid: 'CA...b0', + To: '+1571XXXXXXX', + CallerZip: '', + ToCountry: 'US', + StirVerstat: 'TN-Validation-Passed-C', + CallToken: '%7B...', + CalledZip: '', + ApiVersion: '2010-04-01', + CalledCity: '', + CallStatus: 'ringing', + From: '+919XXXXXXXXX', + AccountSid: 'A....1c', + CalledCountry: 'US', + CallerCity: '', + ToCity: '', + FromCountry: 'IN', + Caller: '+919XXXXXXXXX' + FromCity: '', + CalledState: 'VA', + FromZip: '', + FromState: 'PXXXXXXX' + } + */ + // parse and validate form data + const formData = await request.formData(); + logger.log('request body:', JSON.stringify(Object.fromEntries(formData))); + const data = ZStandardRequestParams.parse(Object.fromEntries(formData)); + logger = logger.child(data.To); + + // get a matching twilio config for this phone number. + // if not found, reject the call + const twilioConfig = await twilioConfigsCollection.findOne({ + phone_number: data.To, + status: 'active', + }); + if (!twilioConfig) { + logger.log('No active twilio config found for this phone number'); + return reject('rejected'); + } + + // extract workflow and project id and fetch workflow from db + // if workflow not found, reject the call + const projectId = twilioConfig.project_id; + const workflowId = twilioConfig.workflow_id; + const workflow = await agentWorkflowsCollection.findOne({ + projectId: projectId, + _id: new ObjectId(workflowId), + }); + if (!workflow) { + logger.log(`Workflow ${workflowId} not found for project ${projectId}`); + return reject('rejected'); + } + + // fetch project tools + const projectTools = await collectProjectTools(projectId); + + // this is the first turn, get the initial assistant response + // and validate it + const { messages } = await getResponse(workflow, projectTools, []); + if (messages.length === 0) { + logger.log('Agent response is empty'); + return hangup(); + } + const lastMessage = messages[messages.length - 1]; + if (lastMessage.role !== 'assistant' || !lastMessage.content) { + logger.log('Invalid last message'); + return hangup(); + } + + // save call state + const call: z.infer = { + callSid: data.CallSid, + to: data.To, + from: data.From, + projectId, + workflowId, + messages, + createdAt: recvdAt.toISOString(), + lastUpdatedAt: new Date().toISOString(), + }; + await twilioInboundCallsCollection.insertOne(call); + + // speak out response + const response = new VoiceResponse(); + response.say(lastMessage.content); + response.gather({ + input: ['speech'], + speechTimeout: 'auto', + language: 'en-US', + enhanced: true, + speechModel: 'phone_call', + action: `/api/twilio/turn/${data.CallSid}`, + }); + return XmlResponse(response); +} \ No newline at end of file diff --git a/apps/rowboat/app/api/twilio/turn/[callSid]/route.ts b/apps/rowboat/app/api/twilio/turn/[callSid]/route.ts new file mode 100644 index 00000000..a0832b4b --- /dev/null +++ b/apps/rowboat/app/api/twilio/turn/[callSid]/route.ts @@ -0,0 +1,97 @@ +import { getResponse } from "@/app/lib/agents"; +import { agentWorkflowsCollection, twilioConfigsCollection, twilioInboundCallsCollection } from "@/app/lib/mongodb"; +import { collectProjectTools } from "@/app/lib/project_tools"; +import { PrefixLogger } from "@/app/lib/utils"; +import VoiceResponse from "twilio/lib/twiml/VoiceResponse"; +import { ObjectId } from "mongodb"; +import { z } from "zod"; +import { hangup, XmlResponse, ZStandardRequestParams } from "../../utils"; +import { Message } from "@/app/lib/types/types"; + +const ZRequestData = ZStandardRequestParams.extend({ + SpeechResult: z.string(), + Confidence: z.string(), +}); + +export async function POST( + request: Request, + { params }: { params: Promise<{ callSid: string }> } +) { + const { callSid } = await params; + let logger = new PrefixLogger(`turn:${callSid}`); + logger.log("Received turn"); + + // parse and validate form data + const formData = await request.formData(); + logger.log('request body:', JSON.stringify(Object.fromEntries(formData))); + const data = ZRequestData.parse(Object.fromEntries(formData)); + + // get call state from db + // if not found, hangup the call + const call = await twilioInboundCallsCollection.findOne({ + callSid, + }); + if (!call) { + logger.log('Call not found'); + return hangup(); + } + const { workflowId, projectId } = call; + + // fetch workflow + const workflow = await agentWorkflowsCollection.findOne({ + projectId: projectId, + _id: new ObjectId(workflowId), + }); + if (!workflow) { + logger.log(`Workflow ${workflowId} not found for project ${projectId}`); + return hangup(); + } + + // fetch project tools + const projectTools = await collectProjectTools(projectId); + + // add user speech as user message, and get assistant response + const reqMessages: z.infer[] = [ + ...call.messages, + { + role: 'user', + content: data.SpeechResult, + } + ]; + const { messages } = await getResponse(workflow, projectTools, reqMessages); + if (messages.length === 0) { + logger.log('Agent response is empty'); + return hangup(); + } + const lastMessage = messages[messages.length - 1]; + if (lastMessage.role !== 'assistant' || !lastMessage.content) { + logger.log('Invalid last message'); + return hangup(); + } + + // save call state + await twilioInboundCallsCollection.updateOne({ + _id: call._id, + }, { + $set: { + messages: [ + ...reqMessages, + ...messages, + ], + lastUpdatedAt: new Date().toISOString(), + } + }); + + // speak out response + const response = new VoiceResponse(); + response.say(lastMessage.content); + response.gather({ + input: ['speech'], + speechTimeout: 'auto', + language: 'en-US', + enhanced: true, + speechModel: 'phone_call', + action: `/api/twilio/turn/${callSid}`, + }); + return XmlResponse(response); +} \ No newline at end of file diff --git a/apps/rowboat/app/api/twilio/utils.ts b/apps/rowboat/app/api/twilio/utils.ts new file mode 100644 index 00000000..4285ddbb --- /dev/null +++ b/apps/rowboat/app/api/twilio/utils.ts @@ -0,0 +1,32 @@ +import TwiML from "twilio/lib/twiml/TwiML"; +import VoiceResponse from "twilio/lib/twiml/VoiceResponse"; +import { z } from "zod"; + +export function XmlResponse(content: TwiML) { + return new Response(content.toString(), { + headers: { + "Content-Type": "text/xml", + }, + }); +} + +export function reject(reason: VoiceResponse.RejectAttributes['reason']) { + return XmlResponse(new VoiceResponse() + .reject({ + reason, + }) + ); +} + +export function hangup() { + return XmlResponse(new VoiceResponse() + .hangup() + ); +} + +export const ZStandardRequestParams = z.object({ + To: z.string(), + Direction: z.literal('inbound'), + CallSid: z.string(), + From: z.string(), +}); \ No newline at end of file diff --git a/apps/rowboat/app/api/uploads/[fileId]/route.ts b/apps/rowboat/app/api/uploads/[fileId]/route.ts index eef03da7..1e76223e 100644 --- a/apps/rowboat/app/api/uploads/[fileId]/route.ts +++ b/apps/rowboat/app/api/uploads/[fileId]/route.ts @@ -8,10 +8,8 @@ import { ObjectId } from 'mongodb'; const UPLOADS_DIR = process.env.RAG_UPLOADS_DIR || '/uploads'; // PUT endpoint to handle file uploads -export async function PUT( - request: NextRequest, - { params }: { params: { fileId: string } } -) { +export async function PUT(request: NextRequest, props: { params: Promise<{ fileId: string }> }) { + const params = await props.params; const fileId = params.fileId; if (!fileId) { return NextResponse.json({ error: 'Missing file ID' }, { status: 400 }); @@ -34,10 +32,8 @@ export async function PUT( } // GET endpoint to handle file downloads -export async function GET( - request: NextRequest, - { params }: { params: { fileId: string } } -) { +export async function GET(request: NextRequest, props: { params: Promise<{ fileId: string }> }) { + const params = await props.params; const fileId = params.fileId; if (!fileId) { return NextResponse.json({ error: 'Missing file ID' }, { status: 400 }); diff --git a/apps/rowboat/app/api/v1/[projectId]/chat/route.ts b/apps/rowboat/app/api/v1/[projectId]/chat/route.ts index 2b1c3029..09659cf6 100644 --- a/apps/rowboat/app/api/v1/[projectId]/chat/route.ts +++ b/apps/rowboat/app/api/v1/[projectId]/chat/route.ts @@ -4,14 +4,13 @@ import { z } from "zod"; import { ObjectId } from "mongodb"; import { authCheck } from "../../utils"; import { ApiRequest, ApiResponse } from "../../../../lib/types/types"; -import { AgenticAPIChatRequest, convertFromAgenticApiToApiMessages, convertFromApiToAgenticApiMessages, convertWorkflowToAgenticAPI } from "../../../../lib/types/agents_api_types"; -import { getAgenticApiResponse } from "../../../../lib/utils"; import { check_query_limit } from "../../../../lib/rate_limiting"; import { PrefixLogger } from "../../../../lib/utils"; import { TestProfile } from "@/app/lib/types/testing_types"; -import { fetchProjectMcpTools } from "@/app/lib/project_tools"; +import { collectProjectTools } from "@/app/lib/project_tools"; import { authorize, getCustomerIdForProject, logUsage } from "@/app/lib/billing"; import { USE_BILLING } from "@/app/lib/feature_flags"; +import { getResponse } from "@/app/lib/agents"; // get next turn / agent response export async function POST( @@ -52,7 +51,7 @@ export async function POST( return Response.json({ error: `Invalid request body: ${result.error.message}` }, { status: 400 }); } const reqMessages = result.data.messages; - const reqState = result.data.state; + const mockToolOverrides = result.data.mockTools; // fetch published workflow id const project = await projectsCollection.findOne({ @@ -64,7 +63,7 @@ export async function POST( } // fetch project tools - const projectTools = await fetchProjectMcpTools(projectId); + const projectTools = await collectProjectTools(projectId); // if workflow id is provided in the request, use it, else use the published workflow id let workflowId = result.data.workflowId ?? project.publishedWorkflowId; @@ -82,6 +81,11 @@ export async function POST( return Response.json({ error: "Workflow not found" }, { status: 404 }); } + // override mock instructions + if (mockToolOverrides) { + workflow.mockTools = mockToolOverrides; + } + // check billing authorization if (USE_BILLING && billingCustomerId) { const agentModels = workflow.agents.reduce((acc, agent) => { @@ -112,34 +116,12 @@ export async function POST( } } - let currentState: unknown = reqState ?? { last_agent_name: workflow.agents[0].name }; - // get assistant response - const { agents, tools, prompts, startAgent } = convertWorkflowToAgenticAPI(workflow, projectTools); - const request: z.infer = { - projectId, - messages: convertFromApiToAgenticApiMessages(reqMessages), - state: currentState, - agents, - tools, - prompts, - startAgent, - testProfile: testProfile ?? undefined, - mcpServers: (project.mcpServers ?? []).map(server => ({ - name: server.name, - serverUrl: server.serverUrl ?? '', - isReady: server.isReady ?? false - })), - toolWebhookUrl: project.webhookUrl ?? '', - }; - - const { messages: agenticMessages, state } = await getAgenticApiResponse(request); - const newMessages = convertFromAgenticApiToApiMessages(agenticMessages); - const newState = state; + const { messages } = await getResponse(workflow, projectTools, reqMessages); // log billing usage if (USE_BILLING && billingCustomerId) { - const agentMessageCount = newMessages.filter(m => m.role === 'assistant').length; + const agentMessageCount = messages.filter(m => m.role === 'assistant').length; await logUsage(billingCustomerId, { type: 'agent_messages', amount: agentMessageCount, @@ -147,8 +129,7 @@ export async function POST( } const responseBody: z.infer = { - messages: newMessages, - state: newState, + messages, }; return Response.json(responseBody); }); diff --git a/apps/rowboat/app/api/widget/v1/chats/[chatId]/close/route.ts b/apps/rowboat/app/api/widget/v1/chats/[chatId]/close/route.ts index d62a4423..e64f347f 100644 --- a/apps/rowboat/app/api/widget/v1/chats/[chatId]/close/route.ts +++ b/apps/rowboat/app/api/widget/v1/chats/[chatId]/close/route.ts @@ -3,10 +3,8 @@ import { chatsCollection } from "../../../../../../lib/mongodb"; import { ObjectId } from "mongodb"; import { authCheck } from "../../../utils"; -export async function POST( - request: NextRequest, - { params }: { params: { chatId: string } } -): Promise { +export async function POST(request: NextRequest, props: { params: Promise<{ chatId: string }> }): Promise { + const params = await props.params; return await authCheck(request, async (session) => { const { chatId } = params; diff --git a/apps/rowboat/app/api/widget/v1/chats/[chatId]/messages/route.ts b/apps/rowboat/app/api/widget/v1/chats/[chatId]/messages/route.ts index 0ce24e49..fb05ffb6 100644 --- a/apps/rowboat/app/api/widget/v1/chats/[chatId]/messages/route.ts +++ b/apps/rowboat/app/api/widget/v1/chats/[chatId]/messages/route.ts @@ -6,10 +6,8 @@ import { Filter, ObjectId } from "mongodb"; import { authCheck } from "../../../utils"; // list messages -export async function GET( - req: NextRequest, - { params }: { params: { chatId: string } } -): Promise { +export async function GET(req: NextRequest, props: { params: Promise<{ chatId: string }> }): Promise { + const params = await props.params; return await authCheck(req, async (session) => { const { chatId } = params; diff --git a/apps/rowboat/app/api/widget/v1/chats/[chatId]/turn/route.ts b/apps/rowboat/app/api/widget/v1/chats/[chatId]/turn/route.ts index 620a9e07..8b4e98ed 100644 --- a/apps/rowboat/app/api/widget/v1/chats/[chatId]/turn/route.ts +++ b/apps/rowboat/app/api/widget/v1/chats/[chatId]/turn/route.ts @@ -4,16 +4,108 @@ import { agentWorkflowsCollection, projectsCollection, chatsCollection, chatMess import { z } from "zod"; import { ObjectId, WithId } from "mongodb"; import { authCheck } from "../../../utils"; -import { convertFromAgenticAPIChatMessages } from "../../../../../../lib/types/agents_api_types"; -import { convertToAgenticAPIChatMessages } from "../../../../../../lib/types/agents_api_types"; -import { convertWorkflowToAgenticAPI } from "../../../../../../lib/types/agents_api_types"; -import { AgenticAPIChatRequest } from "../../../../../../lib/types/agents_api_types"; -import { getAgenticApiResponse } from "../../../../../../lib/utils"; import { check_query_limit } from "../../../../../../lib/rate_limiting"; import { PrefixLogger } from "../../../../../../lib/utils"; -import { fetchProjectMcpTools } from "@/app/lib/project_tools"; +import { collectProjectTools } from "@/app/lib/project_tools"; import { authorize, getCustomerIdForProject, logUsage } from "@/app/lib/billing"; import { USE_BILLING } from "@/app/lib/feature_flags"; +import { getResponse } from "@/app/lib/agents"; +import { Message, AssistantMessage, AssistantMessageWithToolCalls, ToolMessage } from "@/app/lib/types/types"; + +function convert(messages: z.infer[]): z.infer[] { + const result: z.infer[] = []; + for (const m of messages) { + if (m.role === 'assistant') { + if ('tool_calls' in m) { + result.push({ + role: 'assistant', + content: null, + agentName: m.agenticSender ?? '', + toolCalls: m.tool_calls.map((t: any) => ({ + function: { + name: t.function.name, + arguments: t.function.arguments, + }, + type: 'function', + id: t.id, + })), + }); + } else { + result.push({ + role: 'assistant', + content: m.content, + agentName: m.agenticSender ?? '', + responseType: m.agenticResponseType, + }); + } + } else if (m.role === 'tool') { + result.push({ + role: 'tool', + content: m.content, + toolCallId: m.tool_call_id, + toolName: m.tool_name, + }); + } else if (m.role === 'system') { + result.push({ + role: 'system', + content: m.content, + }); + } else if (m.role === 'user') { + result.push({ + role: 'user', + content: m.content, + }); + } + } + return result; +} + +function convertBack(messages: z.infer[]): z.infer[] { + const result: z.infer[] = []; + for (const m of messages) { + if (m.role === 'assistant') { + if ('toolCalls' in m) { + result.push({ + version: 'v1', + chatId: '', + createdAt: new Date().toISOString(), + role: 'assistant', + agenticSender: m.agentName, + agenticResponseType: 'external', + tool_calls: m.toolCalls.map((t: any) => ({ + function: { + name: t.function.name, + arguments: t.function.arguments, + }, + type: 'function', + id: t.id, + })), + }); + } else { + result.push({ + version: 'v1', + chatId: '', + createdAt: new Date().toISOString(), + role: 'assistant', + content: m.content, + agenticSender: m.agentName, + agenticResponseType: m.responseType, + }); + } + } else if (m.role === 'tool') { + result.push({ + version: 'v1', + chatId: '', + createdAt: new Date().toISOString(), + role: 'tool', + content: m.content, + tool_call_id: m.toolCallId, + tool_name: m.toolName, + }); + } + } + return result; +} // get next turn / agent response export async function POST( @@ -90,7 +182,7 @@ export async function POST( } // fetch project tools - const projectTools = await fetchProjectMcpTools(session.projectId); + const projectTools = await collectProjectTools(session.projectId); // fetch workflow const workflow = await agentWorkflowsCollection.findOne({ @@ -119,47 +211,23 @@ export async function POST( } // get assistant response - const { agents, tools, prompts, startAgent } = convertWorkflowToAgenticAPI(workflow, projectTools); - const unsavedMessages: z.infer[] = [userMessage]; - let state: unknown = chat.agenticState ?? { last_agent_name: startAgent }; + const inMessages: z.infer[] = convert(messages); + inMessages.push(userMessage); - const request: z.infer = { - projectId: session.projectId, - messages: convertToAgenticAPIChatMessages([systemMessage, ...messages, ...unsavedMessages]), - state, - agents, - tools, - prompts, - startAgent, - mcpServers: (projectSettings.mcpServers ?? []).map(server => ({ - name: server.name, - serverUrl: server.serverUrl || '', - isReady: server.isReady - })), - toolWebhookUrl: projectSettings.webhookUrl ?? '', - testProfile: undefined, - }; - logger.log(`Sending agentic request`); - const response = await getAgenticApiResponse(request); - state = response.state; - if (response.messages.length === 0) { - throw new Error("No messages returned from assistant"); - } - const convertedMessages = convertFromAgenticAPIChatMessages(response.messages); - unsavedMessages.push(...convertedMessages.map(m => ({ - ...m, - version: 'v1' as const, - chatId, - createdAt: new Date().toISOString(), - }))); + const { messages: responseMessages } = await getResponse(workflow, projectTools, [systemMessage, ...inMessages]); + const convertedResponseMessages = convertBack(responseMessages); + const unsavedMessages = [ + userMessage, + ...convertedResponseMessages, + ]; logger.log(`Saving ${unsavedMessages.length} new messages and updating chat state`); await chatMessagesCollection.insertMany(unsavedMessages); - await chatsCollection.updateOne({ _id: new ObjectId(chatId) }, { $set: { agenticState: state } }); + await chatsCollection.updateOne({ _id: new ObjectId(chatId) }, { $set: { agenticState: chat.agenticState } }); // log billing usage if (USE_BILLING && billingCustomerId) { - const agentMessageCount = convertedMessages.filter(m => m.role === 'assistant').length; + const agentMessageCount = convertedResponseMessages.filter(m => m.role === 'assistant').length; await logUsage(billingCustomerId, { type: 'agent_messages', amount: agentMessageCount, diff --git a/apps/rowboat/app/app.tsx b/apps/rowboat/app/app.tsx index 549d8637..ef2458ea 100644 --- a/apps/rowboat/app/app.tsx +++ b/apps/rowboat/app/app.tsx @@ -1,23 +1,21 @@ 'use client'; -import { TypewriterEffect } from "./lib/components/typewriter"; import Image from 'next/image'; import logo from "@/public/logo.png"; -import { useUser } from "@auth0/nextjs-auth0/client"; +import { useUser } from "@auth0/nextjs-auth0"; import { useRouter } from "next/navigation"; import { Spinner } from "@heroui/react"; -import { LogInIcon } from "lucide-react"; export function App() { const router = useRouter(); - const { user, error, isLoading } = useUser(); + const { user, isLoading } = useUser(); if (user) { router.push("/projects"); } // Add auto-redirect for non-authenticated users - if (!isLoading && !user && !error) { - router.push("/api/auth/login"); + if (!isLoading && !user) { + router.push("/auth/login"); } return ( @@ -30,8 +28,7 @@ export function App() { alt="RowBoat Logo" height={40} /> - {(isLoading || (!user && !error)) && } - {error &&
{error.message}
} + {(isLoading || !user) && } {user &&
Welcome, {user.name}
diff --git a/apps/rowboat/app/billing/callback/page.tsx b/apps/rowboat/app/billing/callback/page.tsx index b5b3bf2a..eae1fa02 100644 --- a/apps/rowboat/app/billing/callback/page.tsx +++ b/apps/rowboat/app/billing/callback/page.tsx @@ -4,13 +4,14 @@ import { redirect } from "next/navigation"; export const dynamic = 'force-dynamic'; -export default async function Page({ - searchParams, -}: { - searchParams: { - redirect: string; +export default async function Page( + props: { + searchParams: Promise<{ + redirect: string; + }> } -}) { +) { + const searchParams = await props.searchParams; const customer = await requireBillingCustomer(); await syncWithStripe(customer._id); const redirectUrl = searchParams.redirect as string; diff --git a/apps/rowboat/app/composio/oauth2/callback/page.tsx b/apps/rowboat/app/composio/oauth2/callback/page.tsx new file mode 100644 index 00000000..b9d9586d --- /dev/null +++ b/apps/rowboat/app/composio/oauth2/callback/page.tsx @@ -0,0 +1,72 @@ +'use client'; + +import { useEffect, useState } from 'react'; +import { CheckCircle, XCircle } from 'lucide-react'; + +export default function OAuth2CallbackPage() { + const [isVisible, setIsVisible] = useState(false); + const [isError, setIsError] = useState(false); + + useEffect(() => { + // Small delay for smooth animation + const timer = setTimeout(() => setIsVisible(true), 100); + + // Check for error parameters in URL + const urlParams = new URLSearchParams(window.location.search); + const error = urlParams.get('error'); + const errorDescription = urlParams.get('error_description'); + + if (error) { + setIsError(true); + } + + // Send message to parent window that OAuth is complete + if (window.opener) { + window.opener.postMessage({ + type: 'OAUTH_COMPLETE', + success: !error, + error: error || null, + errorDescription: errorDescription || null, + timestamp: Date.now() + }, window.location.origin); + + // Close this window after a short delay + setTimeout(() => { + window.close(); + }, 3000); + } + + return () => clearTimeout(timer); + }, []); + + return ( +
+
+
+ {isError ? ( + + ) : ( + + )} +
+ +

+ {isError ? 'OAuth2 Flow Failed' : 'OAuth2 Flow Completed'} +

+ +

+ {isError + ? 'There was an issue with the authentication. Please try again.' + : 'Your authentication was successful. You can safely close this page now.' + } +

+ +
+ This window will automatically close in a few seconds... +
+
+
+ ); +} diff --git a/apps/rowboat/app/globals.css b/apps/rowboat/app/globals.css index 359e21ab..a95e1df0 100644 --- a/apps/rowboat/app/globals.css +++ b/apps/rowboat/app/globals.css @@ -1,7 +1,13 @@ +@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600&display=swap'); +@import 'tailwindcss'; @import './styles/quill-mentions.css'; -@tailwind base; -@tailwind components; -@tailwind utilities; + +@plugin './hero.ts'; + +@source '../node_modules/@heroui/theme/dist/**/*.{js,ts,jsx,tsx}'; +@custom-variant dark (&:is(.dark *)); + +@reference 'tailwindcss'; @layer utilities { .text-balance { @@ -9,11 +15,11 @@ } .custom-scrollbar { scrollbar-width: thin; - scrollbar-color: #E4E4E7 transparent; + scrollbar-color: rgba(156, 163, 175, 0.3) transparent; } .custom-scrollbar::-webkit-scrollbar { - width: 6px; + width: 4px; } .custom-scrollbar::-webkit-scrollbar-track { @@ -21,17 +27,19 @@ } .custom-scrollbar::-webkit-scrollbar-thumb { - background-color: #E4E4E7; - border-radius: 3px; + background-color: rgba(156, 163, 175, 0.3); + border-radius: 4px; + border: none; } /* Dark mode */ .dark .custom-scrollbar { - scrollbar-color: #3F3F46 transparent; + scrollbar-color: rgba(63, 63, 70, 0.4) transparent; } .dark .custom-scrollbar::-webkit-scrollbar-thumb { - background-color: #3F3F46; + background-color: rgba(63, 63, 70, 0.4); + border: none; } } @@ -106,74 +114,61 @@ html, body { input, textarea, select { @apply rounded-lg border-[#E5E7EB] dark:border-[#2E2E30] bg-[#F3F4F6] dark:bg-[#2A2A2D] - focus:ring-2 focus:ring-indigo-500 focus:ring-opacity-50 + focus:ring-2 focus:ring-indigo-500/50 transition-all duration-200; } } -@layer base { - * { - @apply border-border; - } - body { - @apply bg-background text-foreground; - } - - .card-shadow { - @apply shadow-sm dark:shadow-none dark:border-border; - } - - .hover-effect { - @apply hover:bg-accent/10 dark:hover:bg-accent/20 transition-colors; - } - - .border-subtle { - @apply border-border dark:border-border/50; - } - - /* Apply rounded corners to common interactive elements by default */ - button, - input, - textarea, - select, - [role="button"], - .card, - .input, - .select, - .textarea, - .button { - @apply !rounded-lg; - } -} - -* { - -webkit-transition: background-color 0.2s ease-in-out, border-color 0.2s ease-in-out, opacity 0.2s ease-in-out !important; - transition: background-color 0.2s ease-in-out, border-color 0.2s ease-in-out, opacity 0.2s ease-in-out !important; -} - -* { - @apply transition-colors duration-200; -} - -/* Add Inter font */ -@import url('https://fonts.googleapis.com/css2?family=Inter:wght@300;400;500;600&display=swap'); - -/* Set base font */ html { font-family: 'Inter', system-ui, -apple-system, sans-serif; } -@keyframes slideUpAndFade { - from { - opacity: 0; - transform: translateY(10px); - } - to { - opacity: 1; - transform: translateY(0); - } +body { + background: var(--background); + color: var(--foreground); } -.animate-slideUpAndFade { - animation: slideUpAndFade 0.2s ease-out forwards; -} \ No newline at end of file +/* Playground chat custom scrollbar: hide track background and border */ +.playground-scrollbar::-webkit-scrollbar { + width: 4px; + background: transparent !important; +} +.playground-scrollbar::-webkit-scrollbar-track { + background: transparent !important; + border: none !important; + box-shadow: none !important; +} +.playground-scrollbar::-webkit-scrollbar-thumb { + background: #9ca3af; + border-radius: 4px; +} + +.playground-scrollbar { + scrollbar-width: thin; + scrollbar-color: #9ca3af transparent; +} + +@keyframes float { + 0% { transform: translateX(0); } + 50% { transform: translateX(24px); } + 100% { transform: translateX(0); } +} + +@keyframes pulse-mascot { + 0%, 100% { transform: scale(1); } + 50% { transform: scale(1.05); } +} + +/* Combine float (side-to-side) and pulse (scale) */ +.animate-float { + animation: float 5s ease-in-out infinite, pulse-mascot 4s infinite; +} + +/* Feedback modal textarea overrides */ +.feedback-modal textarea, +.feedback-modal textarea:focus { + font-size: 0.75rem !important; /* Tailwind's text-xs */ + box-shadow: none !important; + outline: none !important; + border-color: #d1d5db !important; /* Tailwind's gray-300 */ +} diff --git a/apps/rowboat/app/hero.ts b/apps/rowboat/app/hero.ts new file mode 100644 index 00000000..6dbf0712 --- /dev/null +++ b/apps/rowboat/app/hero.ts @@ -0,0 +1,3 @@ +// hero.ts +import { heroui } from "@heroui/react"; +export default heroui(); \ No newline at end of file diff --git a/apps/rowboat/app/layout.tsx b/apps/rowboat/app/layout.tsx index 8946741a..1f0130bf 100644 --- a/apps/rowboat/app/layout.tsx +++ b/apps/rowboat/app/layout.tsx @@ -1,10 +1,10 @@ import "./globals.css"; import { ThemeProvider } from "./providers/theme-provider"; -import { UserProvider } from '@auth0/nextjs-auth0/client'; import { Inter } from "next/font/google"; import { Providers } from "./providers"; import { Metadata } from "next"; import { HelpModalProvider } from "./providers/help-modal-provider"; +import { Auth0Provider } from "@auth0/nextjs-auth0"; const inter = Inter({ subsets: ["latin"] }); @@ -21,7 +21,7 @@ export default function RootLayout({ children: React.ReactNode; }>) { return - + @@ -31,6 +31,6 @@ export default function RootLayout({ - + ; } diff --git a/apps/rowboat/app/lib/agent_instructions.ts b/apps/rowboat/app/lib/agent_instructions.ts new file mode 100644 index 00000000..ad6567ec --- /dev/null +++ b/apps/rowboat/app/lib/agent_instructions.ts @@ -0,0 +1,132 @@ +/** + * Instructions for agents that use RAG (Retrieval Augmented Generation) + */ +export const RAG_INSTRUCTIONS = (ragToolName: string): string => ` +# Instructions about using the article retrieval tool +- Where relevant, use the articles tool: ${ragToolName} to fetch articles with knowledge relevant to the query and use its contents to respond to the user. +- Do not send a separate message first asking the user to wait while you look up information. Immediately fetch the articles and respond to the user with the answer to their query. +- Do not make up information. If the article's contents do not have the answer, give up control of the chat (or transfer to your parent agent, as per your transfer instructions). Do not say anything to the user. +`; + +/** + * Instructions for child agents that are aware of parent agents + * These instructions guide agents that can transfer control to parent agents + */ +export const TRANSFER_PARENT_AWARE_INSTRUCTIONS = (candidateParentsNameDescriptionTools: string): string => ` +# Instructions about using your parent agents +You have the following candidate parent agents that you can transfer the chat to, using the appropriate tool calls for the transfer: +${candidateParentsNameDescriptionTools}. + +## Notes: +- During runtime, you will be provided with a tool call for exactly one of these parent agents that you can use. Use that tool call to transfer the chat to the parent agent in case you are unable to handle the chat (e.g. if it is not in your scope of instructions). +- Transfer the chat to the appropriate agent, based on the chat history and / or the user's request. +- When you transfer the chat to another agent, you should not provide any response to the user. For example, do not say 'Transferring chat to X agent' or anything like that. Just invoke the tool call to transfer to the other agent. +- Do NOT ever mention the existence of other agents. For example, do not say 'Please check with X agent for details regarding processing times.' or anything like that. +- If any other agent transfers the chat to you without responding to the user, it means that they don't know how to help. Do not transfer the chat to back to the same agent in this case. In such cases, you should transfer to the escalation agent using the appropriate tool call. Never ask the user to contact support. +`; + +/** + * Instructions for child agents that give up control to parent agents + * These instructions guide agents that need to relinquish control to parent agents + */ +export const TRANSFER_GIVE_UP_CONTROL_INSTRUCTIONS = (candidateParentsNameDescriptionTools: string): string => ` +# Instructions about giving up chat control +- If you are unable to handle the chat (e.g. if it is not in your scope of instructions), you should give up control of the chat by calling: ${candidateParentsNameDescriptionTools}. +- If you already have an instruction before this about calling the same agent, you can discard this particular instruction. + +## Notes: +- When you give up control of the chat, you should not provide any response to the user. Just invoke the tool call to give up control. +`; + +/** + * Instructions for parent agents that need to transfer the chat to other specialized (children) agents + * These instructions guide parent agents in delegating tasks to specialized child agents + */ +export const TRANSFER_CHILDREN_INSTRUCTIONS = (otherAgentNameDescriptionsTools: string): string => ` +# Instructions about using other specialized agents +You have the following specialized agents that you can transfer the chat to, using the appropriate tool calls for the transfer: +${otherAgentNameDescriptionsTools} + +## Notes: +- Transfer the chat to the appropriate agent, based on the chat history and / or the user's request. +- When you transfer the chat to another agent, you should not provide any response to the user. For example, do not say 'Transferring chat to X agent' or anything like that. Just invoke the tool call to transfer to the other agent. +- Do NOT ever mention the existence of other agents. For example, do not say 'Please check with X agent for details regarding processing times.' or anything like that. +- If any other agent transfers the chat to you without responding to the user, it means that they don't know how to help. Do not transfer the chat to back to the same agent in this case. In such cases, you should transfer to the escalation agent using the appropriate tool call. Never ask the user to contact support. +`; + +/** + * Additional instruction for escalation agent when called due to an error + * These instructions are used when other agents are unable to handle the chat + */ +export const ERROR_ESCALATION_AGENT_INSTRUCTIONS = ` +# Context +The rest of the parts of the chatbot were unable to handle the chat. Hence, the chat has been escalated to you. In addition to your other instructions, tell the user that you are having trouble handling the chat - say "I'm having trouble helping with your request. Sorry about that.". Remember you are a part of the chatbot as well. +`; + +/** + * Universal system message formatting + * Template for system-wide context and instructions + */ +export const SYSTEM_MESSAGE = (systemMessage: string): string => ` +# Additional System-Wide Context or Instructions: +${systemMessage} +`; + +/** + * Instructions for non-repeat child transfer + * Critical rules for handling agent transfers and handoffs to prevent circular transfers + */ +export const CHILD_TRANSFER_RELATED_INSTRUCTIONS = ` +# Critical Rules for Agent Transfers and Handoffs + +- SEQUENTIAL TRANSFERS AND RESPONSES: + 1. BEFORE transferring to any agent: + - Plan your complete sequence of needed transfers + - Document which responses you need to collect + + 2. DURING transfers: + - Transfer to only ONE agent at a time + - Wait for that agent's COMPLETE response and then proceed with the next agent + - Store the response for later use + - Only then proceed with the next transfer + - Never attempt parallel or simultaneous transfers + - CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output. + + 3. AFTER receiving a response: + - Do not transfer to another agent until you've processed the current response + - If you need to transfer to another agent, wait for your current processing to complete + - Never transfer back to an agent that has already responded + +- COMPLETION REQUIREMENTS: + - Never provide final response until ALL required agents have been consulted + - Never attempt to get multiple responses in parallel + - If a transfer is rejected due to multiple handoffs: + 1. Complete current response processing + 2. Then retry the transfer as next in sequence + 3. Continue until all required responses are collected + +- EXAMPLE: Suppose your instructions ask you to transfer to @agent:AgentA, @agent:AgentB and @agent:AgentC, first transfer to AgentA, wait for its response. Then transfer to AgentB, wait for its response. Then transfer to AgentC, wait for its response. Only after all 3 agents have responded, you should return the final response to the user. +`; + +export const CONVERSATION_TYPE_INSTRUCTIONS = (): string => ` +- You are an agent that is part of a workflow of (one or more) interconnected agents that work together to be an assistant. +- You will be directly interacting with the user. +- It is possible that some other agent might have invoked you to talk to the user. +- Reading the messages in the chat history will give you context about the conversation. But importantly, your response should simply be the direct text to the user. +- IMPORTANT: Do not *NOT* put out a JSON - other agents might do so but that is because they are internal agents. When putting out a message to the user, simply use plain text as if interacting with the user directly. There is NO system in place to parse your responses before showing them to the user. +- Seeing the tool calls that transfer / handoff control will help you understand the flow of the conversation and which agent produced each message. +- When using internal messages that other agents have put out, make sure to write it in a way that is suitable to be shown to the user and in accordance with further instructions below. +- These are high level instructions only. The user will provide more specific instructions which will be below. +`; + +export const TASK_TYPE_INSTRUCTIONS = (): string => ` +- You are an agent that is part of a workflow of (one or more) interconnected agents that work together to be an assistant. +- Use the JSON format to convey your responses. The JSON should have 3 keys. +- The first key in the JSON response should be your "thought" - analysizing what has happened till now and what you need to do in this turn. +- The second key should be your "response". While you will put out a message, your response will not be shown directly to the user. Instead, your response will be used by the agent that might have invoked you and (possibly) other agents in the workflow. Therefore, your responses must be worded in such a way that it is useful for other agents and not addressed to the user. +- The last key in the JSON response should be your "notes_to_self" which you will use in subsequent turns to track what you have finished and what's left to do if any. +- IMPORTANT: If you have all the information to take action, such as calling a tool or writing a response, you should do that in the immediate turn. Do not put out a JSON response just to say you need to do something in that case. +- Reading the messages in the chat history will give you context about the conversation. +- Seeing the tool calls that transfer / handoff control will help you understand the flow of the conversation and which agent produced each message. +- These are high level instructions only. The user will provide more specific instructions which will be below. +`; \ No newline at end of file diff --git a/apps/rowboat/app/lib/agents.ts b/apps/rowboat/app/lib/agents.ts new file mode 100644 index 00000000..aa8a482a --- /dev/null +++ b/apps/rowboat/app/lib/agents.ts @@ -0,0 +1,1230 @@ +// External dependencies +import { Agent, AgentInputItem, run, tool, Tool } from "@openai/agents"; +import { RECOMMENDED_PROMPT_PREFIX } from "@openai/agents-core/extensions"; +import { aisdk } from "@openai/agents-extensions"; +import { createOpenAI } from "@ai-sdk/openai"; +import { CoreMessage, embed, generateText } from "ai"; +import { ObjectId } from "mongodb"; +import { z } from "zod"; +import { Composio } from '@composio/core'; +import { SignJWT } from "jose"; +import crypto from "crypto"; + +// Internal dependencies +import { embeddingModel } from '../lib/embedding'; +import { getMcpClient } from "./mcp"; +import { dataSourceDocsCollection, dataSourcesCollection, projectsCollection } from "./mongodb"; +import { qdrantClient } from '../lib/qdrant'; +import { EmbeddingRecord } from "./types/datasource_types"; +import { ConnectedEntity, sanitizeTextWithMentions, Workflow, WorkflowAgent, WorkflowPrompt, WorkflowTool } from "./types/workflow_types"; +import { CHILD_TRANSFER_RELATED_INSTRUCTIONS, CONVERSATION_TYPE_INSTRUCTIONS, RAG_INSTRUCTIONS, TASK_TYPE_INSTRUCTIONS } from "./agent_instructions"; +import { PrefixLogger } from "./utils"; +import { Message, AssistantMessage, AssistantMessageWithToolCalls, ToolMessage } from "./types/types"; + +const PROVIDER_API_KEY = process.env.PROVIDER_API_KEY || process.env.OPENAI_API_KEY || ''; +const PROVIDER_BASE_URL = process.env.PROVIDER_BASE_URL || undefined; +const MODEL = process.env.PROVIDER_DEFAULT_MODEL || 'gpt-4o'; + +const openai = createOpenAI({ + apiKey: PROVIDER_API_KEY, + baseURL: PROVIDER_BASE_URL, +}); + +const ZUsage = z.object({ + tokens: z.object({ + total: z.number(), + prompt: z.number(), + completion: z.number(), + }), +}); + +const ZOutMessage = z.union([ + AssistantMessage, + AssistantMessageWithToolCalls, + ToolMessage, +]); + +// Helper to handle mock tool responses +async function invokeMockTool( + logger: PrefixLogger, + toolName: string, + args: string, + description: string, + mockInstructions: string +): Promise { + logger = logger.child(`invokeMockTool`); + logger.log(`toolName: ${toolName}`); + logger.log(`args: ${args}`); + logger.log(`description: ${description}`); + logger.log(`mockInstructions: ${mockInstructions}`); + + const messages: CoreMessage[] = [{ + role: "system" as const, + content: `You are simulating the execution of a tool called '${toolName}'. Here is the description of the tool: ${description}. Here are the instructions for the mock tool: ${mockInstructions}. Generate a realistic response as if the tool was actually executed with the given parameters.` + }, { + role: "user" as const, + content: `Generate a realistic response for the tool '${toolName}' with these parameters: ${args}. The response should be concise and focused on what the tool would actually return.` + }]; + + const { text } = await generateText({ + model: openai(MODEL), + messages, + }); + logger.log(`generated text: ${text}`); + + return text; +} + +// Helper to handle RAG tool calls +async function invokeRagTool( + logger: PrefixLogger, + projectId: string, + query: string, + sourceIds: string[], + returnType: 'chunks' | 'content', + k: number +): Promise<{ + title: string; + name: string; + content: string; + docId: string; + sourceId: string; +}[]> { + logger = logger.child(`invokeRagTool`); + logger.log(`projectId: ${projectId}`); + logger.log(`query: ${query}`); + logger.log(`sourceIds: ${sourceIds.join(', ')}`); + logger.log(`returnType: ${returnType}`); + logger.log(`k: ${k}`); + + // Create embedding for question + const { embedding } = await embed({ + model: embeddingModel, + value: query, + }); + + // Fetch all data sources for this project + const sources = await dataSourcesCollection.find({ + projectId: projectId, + active: true, + }).toArray(); + const validSourceIds = sources + .filter(s => sourceIds.includes(s._id.toString())) // id should be in sourceIds + .filter(s => s.active) // should be active + .map(s => s._id.toString()); + logger.log(`valid source ids: ${validSourceIds.join(', ')}`); + + // if no sources found, return empty response + if (validSourceIds.length === 0) { + logger.log(`no valid source ids found, returning empty response`); + return []; + } + + // Perform vector search + const qdrantResults = await qdrantClient.query("embeddings", { + query: embedding, + filter: { + must: [ + { key: "projectId", match: { value: projectId } }, + { key: "sourceId", match: { any: validSourceIds } }, + ], + }, + limit: k, + with_payload: true, + }); + logger.log(`found ${qdrantResults.points.length} results`); + + // if return type is chunks, return the chunks + let results = qdrantResults.points.map((point) => { + const { title, name, content, docId, sourceId } = point.payload as z.infer['payload']; + return { + title, + name, + content, + docId, + sourceId, + }; + }); + + if (returnType === 'chunks') { + logger.log(`returning chunks`); + return results; + } + + // otherwise, fetch the doc contents from mongodb + const docs = await dataSourceDocsCollection.find({ + _id: { $in: results.map(r => new ObjectId(r.docId)) }, + }).toArray(); + logger.log(`fetched docs: ${docs.length}`); + + // map the results to the docs + results = results.map(r => { + const doc = docs.find(d => d._id.toString() === r.docId); + return { + ...r, + content: doc?.content || '', + }; + }); + + return results; +} + +async function invokeWebhookTool( + logger: PrefixLogger, + projectId: string, + name: string, + input: any, +): Promise { + logger = logger.child(`invokeWebhookTool`); + logger.log(`projectId: ${projectId}`); + logger.log(`name: ${name}`); + logger.log(`input: ${JSON.stringify(input)}`); + + const project = await projectsCollection.findOne({ + "_id": projectId, + }); + if (!project) { + throw new Error('Project not found'); + } + + if (!project.webhookUrl) { + throw new Error('Webhook URL not found'); + } + + // prepare request body + const toolCall: z.infer[number] = { + id: crypto.randomUUID(), + type: "function", + function: { + name, + arguments: JSON.stringify(input), + }, + } + const content = JSON.stringify({ + toolCall, + }); + const requestId = crypto.randomUUID(); + const bodyHash = crypto + .createHash('sha256') + .update(content, 'utf8') + .digest('hex'); + + // sign request + const jwt = await new SignJWT({ + requestId, + projectId, + bodyHash, + }) + .setProtectedHeader({ + alg: 'HS256', + typ: 'JWT', + }) + .setIssuer('rowboat') + .setAudience(project.webhookUrl) + .setSubject(`tool-call-${toolCall.id}`) + .setJti(requestId) + .setIssuedAt() + .setExpirationTime("5 minutes") + .sign(new TextEncoder().encode(project.secret)); + + // make request + const request = { + requestId, + content, + }; + const response = await fetch(project.webhookUrl, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'x-signature-jwt': jwt, + }, + body: JSON.stringify(request), + }); + if (!response.ok) { + throw new Error(`Failed to call webhook: ${response.status}: ${response.statusText}`); + } + const responseBody = await response.json(); + return responseBody; +} + +// Helper to handle MCP tool calls +async function invokeMcpTool( + logger: PrefixLogger, + projectId: string, + name: string, + input: any, + mcpServerURL: string, + mcpServerName: string +) { + logger = logger.child(`invokeMcpTool`); + logger.log(`projectId: ${projectId}`); + logger.log(`name: ${name}`); + logger.log(`input: ${JSON.stringify(input)}`); + logger.log(`mcpServerURL: ${mcpServerURL}`); + logger.log(`mcpServerName: ${mcpServerName}`); + + const client = await getMcpClient(mcpServerURL, mcpServerName || ''); + const result = await client.callTool({ + name, + arguments: input, + }); + logger.log(`mcp tool result: ${JSON.stringify(result)}`); + await client.close(); + return result; +} + +// Helper to handle composio tool calls +async function invokeComposioTool( + logger: PrefixLogger, + projectId: string, + name: string, + composioData: z.infer['composioData'] & {}, + input: any, +) { + logger = logger.child(`invokeComposioTool`); + logger.log(`projectId: ${projectId}`); + logger.log(`name: ${name}`); + logger.log(`input: ${JSON.stringify(input)}`); + + const { slug, toolkitSlug, noAuth } = composioData; + + let connectedAccountId: string | undefined = undefined; + if (!noAuth) { + const project = await projectsCollection.findOne({ _id: projectId }); + if (!project) { + throw new Error(`project ${projectId} not found`); + } + connectedAccountId = project.composioConnectedAccounts?.[toolkitSlug]?.id; + if (!connectedAccountId) { + throw new Error(`connected account id not found for project ${projectId} and toolkit ${toolkitSlug}`); + } + } + + const composio = new Composio(); + + const result = await composio.tools.execute(slug, { + userId: projectId, + arguments: input, + connectedAccountId: connectedAccountId, + }); + logger.log(`composio tool result: ${JSON.stringify(result)}`); + return result.data; +} + +// Helper to create RAG tool +function createRagTool( + logger: PrefixLogger, + config: z.infer, + projectId: string +): Tool { + if (!config.ragDataSources?.length) { + throw new Error(`data sources not found for agent ${config.name}`); + } + + return tool({ + name: "rag_search", + description: config.description, + parameters: z.object({ + query: z.string().describe("The query to search for") + }), + async execute(input: { query: string }) { + const results = await invokeRagTool( + logger, + projectId, + input.query, + config.ragDataSources || [], + config.ragReturnType || 'chunks', + config.ragK || 3 + ); + return JSON.stringify({ + results, + }); + } + }); +} + +// Helper to create a mock tool +function createMockTool( + logger: PrefixLogger, + config: z.infer, +): Tool { + return tool({ + name: config.name, + description: config.description, + strict: false, + parameters: { + type: 'object', + properties: config.parameters.properties, + required: config.parameters.required || [], + additionalProperties: true, + }, + async execute(input: any) { + try { + const result = await invokeMockTool( + logger, + config.name, + JSON.stringify(input), + config.description, + config.mockInstructions || '' + ); + return JSON.stringify({ + result, + }); + } catch (error) { + logger.log(`Error executing mock tool ${config.name}:`, error); + return JSON.stringify({ + error: `Mock tool execution failed: ${error}`, + }); + } + } + }); +} + +// Helper to create a webhook tool +function createWebhookTool( + logger: PrefixLogger, + config: z.infer, + projectId: string, +): Tool { + const { name, description, parameters } = config; + + return tool({ + name, + description, + strict: false, + parameters: { + type: 'object', + properties: parameters.properties, + required: parameters.required || [], + additionalProperties: true, + }, + async execute(input: any) { + try { + const result = await invokeWebhookTool(logger, projectId, name, input); + return JSON.stringify({ + result, + }); + } catch (error) { + logger.log(`Error executing webhook tool ${config.name}:`, error); + return JSON.stringify({ + error: `Tool execution failed: ${error}`, + }); + } + } + }); +} + +// Helper to create an mcp tool +function createMcpTool( + logger: PrefixLogger, + config: z.infer, + projectId: string +): Tool { + const { name, description, parameters, mcpServerName, mcpServerURL } = config; + + return tool({ + name, + description, + strict: false, + parameters: { + type: 'object', + properties: parameters.properties, + required: parameters.required || [], + additionalProperties: true, + }, + async execute(input: any) { + try { + const result = await invokeMcpTool(logger, projectId, name, input, mcpServerURL || '', mcpServerName || ''); + return JSON.stringify({ + result, + }); + } catch (error) { + logger.log(`Error executing mcp tool ${name}:`, error); + return JSON.stringify({ + error: `Tool execution failed: ${error}`, + }); + } + } + }); +} + +// Helper to create a composio tool +function createComposioTool( + logger: PrefixLogger, + config: z.infer, + projectId: string +): Tool { + const { name, description, parameters, composioData } = config; + + if (!composioData) { + throw new Error(`composio data not found for tool ${name}`); + } + + return tool({ + name, + description, + strict: false, + parameters: { + type: 'object', + properties: parameters.properties, + required: parameters.required || [], + additionalProperties: true, + }, + async execute(input: any) { + try { + const result = await invokeComposioTool(logger, projectId, name, composioData, input); + return JSON.stringify({ + result, + }); + } catch (error) { + logger.log(`Error executing composio tool ${name}:`, error); + return JSON.stringify({ + error: `Tool execution failed: ${error}`, + }); + } + } + }); +} + +// Helper to create an agent +function createAgent( + logger: PrefixLogger, + config: z.infer, + tools: Record, + projectTools: z.infer[], + workflow: z.infer, + promptConfig: Record>, +): { agent: Agent, entities: z.infer[] } { + const agentLogger = logger.child(`createAgent: ${config.name}`); + + // Combine instructions and examples + let instructions = `${RECOMMENDED_PROMPT_PREFIX} + +## Your Name +${config.name} + +## Description +${config.description} + +## About You + +${config.outputVisibility === 'user_facing' ? CONVERSATION_TYPE_INSTRUCTIONS() : TASK_TYPE_INSTRUCTIONS()} + +## Instructions + +${config.instructions} + +${config.examples ? ('# Examples\n' + config.examples) : ''} + +${'-'.repeat(100)} + +${CHILD_TRANSFER_RELATED_INSTRUCTIONS} +`; + + let { sanitized, entities } = sanitizeTextWithMentions(instructions, workflow, projectTools); + agentLogger.log(`instructions: ${JSON.stringify(sanitized)}`); + agentLogger.log(`mentions: ${JSON.stringify(entities)}`); + + // // add prompts to instructions + // for (const e of entities) { + // if (e.type === 'prompt') { + // const prompt = promptConfig[e.name]; + // if (prompt) { + // compiledInstructions = compiledInstructions + '\n\n# ' + prompt.name + '\n' + prompt.prompt; + // } + // } + // } + + const agentTools = entities.filter(e => e.type === 'tool').map(e => tools[e.name]).filter(Boolean) as Tool[]; + + // Add RAG tool if needed + if (config.ragDataSources?.length) { + const ragTool = createRagTool(logger, config, workflow.projectId); + agentTools.push(ragTool); + + // update instructions to include RAG instructions + sanitized = sanitized + '\n\n' + ('-'.repeat(100)) + '\n\n' + RAG_INSTRUCTIONS(ragTool.name); + agentLogger.log(`added rag instructions`); + } + + // Create the agent with the dynamic instructions + const agent = new Agent({ + name: config.name, + instructions: sanitized, + tools: agentTools, + model: aisdk(openai(config.model)), + // model: config.model, + modelSettings: { + temperature: 0.0, + } + }); + agentLogger.log(`created agent`); + + return { + agent, + entities, + }; +} + +// Convert messages to agent input items +function convertMsgsInput(messages: z.infer[]): AgentInputItem[] { + const msgs: AgentInputItem[] = []; + + for (const msg of messages) { + if (msg.role === 'assistant' && msg.content) { + msgs.push({ + role: 'assistant', + content: [{ + type: 'output_text', + text: `${msg.content}`, + }], + status: 'completed', + }); + } else if (msg.role === 'user') { + msgs.push({ + role: 'user', + content: msg.content, + }); + } else if (msg.role === 'system') { + msgs.push({ + role: 'system', + content: msg.content, + }); + } + } + + return msgs; +} + +// Helper to determine the next agent name based on control settings +function getStartOfTurnAgentName( + logger: PrefixLogger, + messages: z.infer[], + agentConfig: Record>, + workflow: z.infer, +): string { + + function createAgentCallStack(messages: z.infer[]): string[] { + const stack: string[] = []; + for (const msg of messages) { + if (msg.role === 'assistant' && msg.agentName) { + // skip duplicate entries + if (stack.length > 0 && stack[stack.length - 1] === msg.agentName) { + continue; + } + // add to stack + stack.push(msg.agentName); + } + } + return stack; + } + + logger = logger.child(`getStartOfTurnAgentName`); + const startAgentStack = createAgentCallStack(messages); + logger.log(`startAgentStack: ${JSON.stringify(startAgentStack)}`); + + // if control type is retain, return last agent + const lastAgentName = startAgentStack.pop() || workflow.startAgent; + logger.log(`setting last agent name initially to: ${lastAgentName}`); + const lastAgentConfig = agentConfig[lastAgentName]; + if (!lastAgentConfig) { + logger.log(`last agent ${lastAgentName} not found in agent config, returning start agent: ${workflow.startAgent}`); + return workflow.startAgent; + } + switch (lastAgentConfig.controlType) { + case 'retain': + logger.log(`last agent ${lastAgentName} control type is retain, returning last agent: ${lastAgentName}`); + return lastAgentName; + case 'relinquish_to_parent': + const parentAgentName = startAgentStack.pop() || workflow.startAgent; + if (startAgentStack.length > 0) { + logger.log(`popped agent from stack: ${lastAgentName} || reason: relinquish to parent triggered`); + } else { + logger.log(`using start agent: ${lastAgentName} || reason: empty stack`); + } + logger.log(`last agent ${lastAgentName} control type is relinquish_to_parent, returning most recent parent: ${parentAgentName}`); + return parentAgentName; + case 'relinquish_to_start': + logger.log(`last agent ${lastAgentName} control type is relinquish_to_start, returning start agent: ${workflow.startAgent}`); + return workflow.startAgent; + } +} + +// Logs an event and then yields it +async function* emitEvent( + logger: PrefixLogger, + event: z.infer | z.infer, +): AsyncIterable | z.infer> { + logger.log(`-> emitting event: ${JSON.stringify(event)}`); + yield event; + return; +} + +// Emits an agent -> agent transfer event +function createTransferEvents( + fromAgent: string, + toAgent: string, +): [z.infer, z.infer] { + const toolCallId = crypto.randomUUID(); + const m1: z.infer = { + role: 'assistant', + content: null, + toolCalls: [{ + id: toolCallId, + type: 'function', + function: { + name: 'transfer_to_agent', + arguments: JSON.stringify({ assistant: toAgent }), + }, + }], + agentName: fromAgent, + }; + + const m2: z.infer = { + role: 'tool', + content: JSON.stringify({ assistant: toAgent }), + toolCallId: toolCallId, + toolName: 'transfer_to_agent', + }; + + return [m1, m2]; +} + +// Tracks agent to agent transfer counts +class AgentTransferCounter { + private calls: Record = {}; + + increment(fromAgent: string, toAgent: string): void { + const key = `${fromAgent}:${toAgent}`; + this.calls[key] = (this.calls[key] || 0) + 1; + } + + get(fromAgent: string, toAgent: string): number { + const key = `${fromAgent}:${toAgent}`; + return this.calls[key] || 0; + } +} + +class UsageTracker { + private usage: { + total: number; + prompt: number; + completion: number; + } = { total: 0, prompt: 0, completion: 0 }; + + increment(total: number, prompt: number, completion: number): void { + this.usage.total += total; + this.usage.prompt += prompt; + this.usage.completion += completion; + } + + get(): { total: number, prompt: number, completion: number } { + return this.usage; + } + + asEvent(): z.infer { + return { + tokens: this.usage, + }; + } +} + +function ensureSystemMessage(logger: PrefixLogger, messages: z.infer[]) { + logger = logger.child(`ensureSystemMessage`); + + // ensure that a system message is set + if (messages.length > 0 && messages[0]?.role !== 'system') { + messages.unshift({ + role: 'system', + content: 'You are a helpful assistant.', + }); + logger.log(`added system message: ${messages[0]?.content}`); + } + + // ensure that system message isn't blank + if (messages.length > 0 && messages[0]?.role === 'system' && !messages[0].content) { + const defaultContext = `You are a helpful assistant. + +Basic context: + - Today's date is ${new Date().toLocaleDateString()} + - Current time is ${new Date().toLocaleTimeString()}.`; + + messages[0].content = defaultContext; + logger.log(`updated system message with default context: ${messages[0].content}`); + } +} + +function mapConfig(workflow: z.infer, projectTools: z.infer[]): { + agentConfig: Record>; + toolConfig: Record>; + promptConfig: Record>; +} { + const agentConfig: Record> = workflow.agents.reduce((acc, agent) => ({ + ...acc, + [agent.name]: agent + }), {}); + const toolConfig: Record> = [ + ...workflow.tools, + ...projectTools, + ].reduce((acc, tool) => ({ + ...acc, + [tool.name]: tool + }), {}); + const promptConfig: Record> = workflow.prompts.reduce((acc, prompt) => ({ + ...acc, + [prompt.name]: prompt + }), {}); + return { agentConfig, toolConfig, promptConfig }; +} + +async function* emitGreetingTurn(logger: PrefixLogger, workflow: z.infer): AsyncIterable | z.infer> { + // find the greeting prompt + const prompt = workflow.prompts.find(p => p.type === 'greeting')?.prompt || 'How can I help you today?'; + logger.log(`greeting turn: ${prompt}`); + + // emit greeting turn + yield* emitEvent(logger, { + role: 'assistant', + content: prompt, + agentName: workflow.startAgent, + responseType: 'external', + }); + + // emit final usage information + yield* emitEvent(logger, new UsageTracker().asEvent()); +} + +function createTools(logger: PrefixLogger, workflow: z.infer, toolConfig: Record>): Record { + const tools: Record = {}; + for (const [toolName, config] of Object.entries(toolConfig)) { + if (workflow.mockTools?.[toolName]) { + tools[toolName] = createMockTool(logger, { + ...config, + mockInstructions: workflow.mockTools?.[toolName], // override mock instructions + }); + logger.log(`created mock tool: ${toolName}`); + } else if (config.isMcp) { + tools[toolName] = createMcpTool(logger, config, workflow.projectId); + logger.log(`created mcp tool: ${toolName}`); + } else if (config.isComposio) { + tools[toolName] = createComposioTool(logger, config, workflow.projectId); + logger.log(`created composio tool: ${toolName}`); + } else if (config.mockTool) { + tools[toolName] = createMockTool(logger, config); + logger.log(`created mock tool: ${toolName}`); + } else { + tools[toolName] = createWebhookTool(logger, config, workflow.projectId); + logger.log(`created webhook tool: ${toolName}`); + } + } + return tools; +} + +function createAgents( + logger: PrefixLogger, + workflow: z.infer, + agentConfig: Record>, + tools: Record, + projectTools: z.infer[], + promptConfig: Record>, +): { agents: Record, mentions: Record[]>, originalInstructions: Record, originalHandoffs: Record } { + const agents: Record = {}; + const mentions: Record[]> = {}; + const originalInstructions: Record = {}; + const originalHandoffs: Record = {}; + + // create agents + for (const [agentName, config] of Object.entries(agentConfig)) { + const { agent, entities } = createAgent( + logger, + config, + tools, + projectTools, + workflow, + promptConfig, + ); + agents[agentName] = agent; + mentions[agentName] = entities; + originalInstructions[agentName] = agent.instructions as string; + // handoffs will be set after all agents are created + } + + // set handoffs + for (const [agentName, agent] of Object.entries(agents)) { + const connectedAgentNames = (mentions[agentName] || []).filter(e => e.type === 'agent').map(e => e.name); + // Only store Agent objects in handoffs (filter out Handoff if present) + agent.handoffs = connectedAgentNames.map(e => agents[e]).filter(Boolean) as Agent[]; + originalHandoffs[agentName] = agent.handoffs.filter(h => h instanceof Agent); + logger.log(`set handoffs for ${agentName}: ${JSON.stringify(connectedAgentNames)}`); + } + + return { agents, mentions, originalInstructions, originalHandoffs }; +} + +// Helper to get give up control instructions for child agents +function getGiveUpControlInstructions( + agent: Agent, + parentAgentName: string, + logger: PrefixLogger +): string { + let dynamicInstructions: string; + if (typeof agent.instructions === 'string') { + dynamicInstructions = agent.instructions; + } else { + throw new Error('Agent instructions must be a string for dynamic injection.'); + } + // Only include the @mention for the parent, not the tool call format + const parentBlock = `@agent:${parentAgentName}`; + // Import the template + const { TRANSFER_GIVE_UP_CONTROL_INSTRUCTIONS } = require('./agent_instructions'); + dynamicInstructions = dynamicInstructions + '\n\n' + TRANSFER_GIVE_UP_CONTROL_INSTRUCTIONS(parentBlock); + // For tracking + logger.log(`Added give up control instructions for ${agent.name} with parent ${parentAgentName}`); + return dynamicInstructions; +} + +// Helper to dynamically inject give up control instructions and handoff +function maybeInjectGiveUpControlInstructions( + agents: Record, + agentConfig: Record>, + childAgentName: string, + parentAgentName: string, + logger: PrefixLogger, + originalInstructions: Record, + originalHandoffs: Record +) { + // Reset to original before injecting + agents[childAgentName].instructions = originalInstructions[childAgentName]; + agents[childAgentName].handoffs = [...originalHandoffs[childAgentName]]; + + const agentConfigObj = agentConfig[childAgentName]; + const isInternal = agentConfigObj?.outputVisibility === 'internal'; + const isRetain = agentConfigObj?.controlType === 'retain'; + const injectLogger = logger.child(`inject`); + injectLogger.log(`isInternal: ${isInternal}`); + injectLogger.log(`isRetain: ${isRetain}`); + if (!isInternal && isRetain) { + // inject give up control instructions + agents[childAgentName].instructions = getGiveUpControlInstructions(agents[childAgentName], parentAgentName, injectLogger); + injectLogger.log(`Added give up control instructions for ${childAgentName} with parent ${parentAgentName}`); + // add the parent agent to the handoff list if not already present + if (!agents[childAgentName].handoffs.includes(agents[parentAgentName])) { + agents[childAgentName].handoffs.push(agents[parentAgentName]); + } + injectLogger.log(`Added parent ${parentAgentName} to handoffs for ${childAgentName}`); + } +} + +// Main function to stream an agentic response +// using OpenAI Agents SDK +export async function* streamResponse( + workflow: z.infer, + projectTools: z.infer[], + messages: z.infer[], +): AsyncIterable | z.infer> { + // Divider log for tracking agent loop start + console.log('-------------------- AGENT LOOP START --------------------'); + // set up logging + let logger = new PrefixLogger(`agent-loop`) + logger.log('projectId', workflow.projectId); + logger.log('workflow', workflow.name); + + // ensure valid system message + ensureSystemMessage(logger, messages); + + // if there is only a system message, emit greeting turn and return + if (messages.length === 1 && messages[0]?.role === 'system') { + yield* emitGreetingTurn(logger, workflow); + return; + } + + // create map of agent, tool and prompt configs + const { agentConfig, toolConfig, promptConfig } = mapConfig(workflow, projectTools); + + + const stack: string[] = []; + logger.log(`initialized stack: ${JSON.stringify(stack)}`); + + // create tools + const tools = createTools(logger, workflow, toolConfig); + + // create agents + const { agents, originalInstructions, originalHandoffs } = createAgents(logger, workflow, agentConfig, tools, projectTools, promptConfig); + + // track agent to agent calls + const transferCounter = new AgentTransferCounter(); + + // track usage + const usageTracker = new UsageTracker(); + + // get next agent name + let agentName = getStartOfTurnAgentName(logger, messages, agentConfig, workflow); + + // set up initial state for loop + logger.log('@@ starting agent turn @@'); + let iter = 0; + const turnMsgs: z.infer[] = [...messages]; + + // loop indefinitely + turnLoop: while (true) { + + logger.log(`starting turn loop iteration: ${iter}`); + // increment loop counter + iter++; + + // set up logging + const loopLogger = logger.child(`iter-${iter}`); + + // log agent info + loopLogger.log(`agent name: ${agentName}`); + loopLogger.log(`stack: ${JSON.stringify(stack)}`); + if (!agents[agentName]) { + throw new Error(`agent not found in agent config!`); + } + const agent: Agent = agents[agentName]!; + + // convert messages to agents sdk compatible input + const inputs = convertMsgsInput(turnMsgs); + + // run the agent + const result = await run( + agent, + inputs, + { + stream: true, + } + ); + + // handle streaming events + for await (const event of result) { + const eventLogger = loopLogger.child(event.type); + + switch (event.type) { + case 'raw_model_stream_event': + if (event.data.type === 'response_done') { + for (const output of event.data.response.output) { + // handle tool call invocation + // except for transfer_to_* tool calls + if (output.type === 'function_call' && !output.name.startsWith('transfer_to')) { + const m: z.infer = { + role: 'assistant', + content: null, + toolCalls: [{ + id: output.callId, + type: 'function', + function: { + name: output.name, + arguments: output.arguments, + }, + }], + agentName: agentName, + }; + + // add message to turn + turnMsgs.push(m); + + // emit event + yield* emitEvent(eventLogger, m); + } + } + + // update usage information + usageTracker.increment( + event.data.response.usage.totalTokens, + event.data.response.usage.inputTokens, + event.data.response.usage.outputTokens + ); + eventLogger.log(`updated usage information: ${JSON.stringify(usageTracker.get())}`); + } + break; + case 'run_item_stream_event': + // handle handoff event + if (event.name === 'handoff_occurred' && event.item.type === 'handoff_output_item') { + // skip if its the same agent + if (agentName === event.item.targetAgent.name) { + eventLogger.log(`skipping handoff to same agent: ${agentName}`); + break; + } + + // Only apply max calls limit to internal agents (task agents) + const targetAgentConfig = agentConfig[event.item.targetAgent.name]; + if (targetAgentConfig?.outputVisibility === 'internal') { + const maxCalls = targetAgentConfig?.maxCallsPerParentAgent || 3; + const currentCalls = transferCounter.get(agentName, event.item.targetAgent.name); + if (currentCalls >= maxCalls) { + eventLogger.log(`skipping handoff to ${event.item.targetAgent.name} || reason: max calls ${maxCalls} exceeded from ${agentName} to internal agent ${event.item.targetAgent.name}`); + continue; + } + } + + // inject give up control instructions if needed (parent handing off to child) + maybeInjectGiveUpControlInstructions( + agents, + agentConfig, + event.item.targetAgent.name, // child + agentName, // parent + eventLogger, + originalInstructions, + originalHandoffs + ); + + // emit transfer tool call invocation + const [transferStart, transferComplete] = createTransferEvents(agentName, event.item.targetAgent.name); + + // add messages to turn + turnMsgs.push(transferStart); + turnMsgs.push(transferComplete); + + // emit events + yield* emitEvent(eventLogger, transferStart); + yield* emitEvent(eventLogger, transferComplete); + + // update transfer counter + transferCounter.increment(agentName, event.item.targetAgent.name); + + const newAgentName = event.item.targetAgent.name; + + loopLogger.log(`switched to agent: ${newAgentName} || reason: handoff by ${agentName}`); + + // add current agent to stack only if new agent is internal + if (agentConfig[newAgentName]?.outputVisibility === 'internal') { + stack.push(agentName); + loopLogger.log(`-- pushed agent to stack: ${agentName} || reason: new agent ${newAgentName} is internal`); + loopLogger.log(`-- stack is now: ${JSON.stringify(stack)}`); + } + + // set this as the new agent name + agentName = newAgentName; + + } + + // handle tool call result + if (event.item.type === 'tool_call_output_item' && + event.item.rawItem.type === 'function_call_result' && + event.item.rawItem.status === 'completed' && + event.item.rawItem.output.type === 'text') { + const m: z.infer = { + role: 'tool', + content: event.item.rawItem.output.text, + toolCallId: event.item.rawItem.callId, + toolName: event.item.rawItem.name, + }; + + // add message to turn + turnMsgs.push(m); + + // emit event + yield* emitEvent(eventLogger, m); + } + + // handle model response message output + if (event.item.type === 'message_output_item' && + event.item.rawItem.type === 'message' && + event.item.rawItem.status === 'completed') { + // check response visibility + const isInternal = agentConfig[agentName]?.outputVisibility === 'internal'; + for (const content of event.item.rawItem.content) { + if (content.type === 'output_text') { + // create message + const msg: z.infer = { + role: 'assistant', + content: content.text, + agentName: agentName, + responseType: isInternal ? 'internal' : 'external', + }; + + // add message to turn + turnMsgs.push(msg); + + // emit event + yield* emitEvent(eventLogger, msg); + } + } + + // if this is an internal agent, switch to previous agent + if (isInternal) { + const current = agentName; + + // if the control type is relinquish_to_parent or retain, we need to pop the stack, else if the control type is relinquish_to_start, we need to use the start agent + if (agentConfig[agentName]?.controlType === 'relinquish_to_parent' || agentConfig[agentName]?.controlType === 'retain') { + agentName = stack.pop()!; + loopLogger.log(`-- popped agent from stack: ${agentName} || reason: ${current} is an internal agent, it put out a message and it has a control type of ${agentConfig[agentName]?.controlType}, hence the flow of control needs to return to the previous agent`); + } else if (agentConfig[agentName]?.controlType === 'relinquish_to_start') { + agentName = workflow.startAgent; + loopLogger.log(`-- using start agent: ${agentName} || reason: ${current} is an internal agent, it put out a message and it has a control type of ${agentConfig[agentName]?.controlType}, hence the flow of control needs to return to the start agent`); + } + + loopLogger.log(`-- stack is now: ${JSON.stringify(stack)}`); + + // emit transfer tool call invocation + const [transferStart, transferComplete] = createTransferEvents(current, agentName); + + // add messages to turn + turnMsgs.push(transferStart); + turnMsgs.push(transferComplete); + + // emit events + yield* emitEvent(eventLogger, transferStart); + yield* emitEvent(eventLogger, transferComplete); + + // update transfer counter + transferCounter.increment(current, agentName); + + // set this as the new agent name + loopLogger.log(`switched to agent: ${agentName} || reason: internal agent (${current}) put out a message`); + + // run the turn from the previous agent + continue turnLoop; + } + break; + } + break; + default: + break; + } + } + + // if the last message was a text response by a user-facing agent, complete the turn + // loopLogger.log(`iter end, turnMsgs: ${JSON.stringify(turnMsgs)}, agentName: ${agentName}`); + const lastMessage = turnMsgs[turnMsgs.length - 1]; + if (agentConfig[agentName]?.outputVisibility === 'user_facing' && + lastMessage?.role === 'assistant' && + lastMessage?.content !== null && + lastMessage?.agentName === agentName + ) { + loopLogger.log(`last message was by a user_facing agent, breaking out of parent loop`); + break turnLoop; + } + } + + // emit usage information + yield* emitEvent(logger, usageTracker.asEvent()); +} + +// this is a sync version of streamResponse +export async function getResponse( + workflow: z.infer, + projectTools: z.infer[], + messages: z.infer[], +): Promise<{ + messages: z.infer[], + usage: z.infer, +}> { + const out: z.infer[] = []; + let usage: z.infer = { + tokens: { + total: 0, + prompt: 0, + completion: 0, + }, + }; + for await (const event of streamResponse(workflow, projectTools, messages)) { + if ('role' in event) { + out.push(event); + } + if ('tokens' in event) { + usage = event; + } + } + return { messages: out, usage }; +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/auth.ts b/apps/rowboat/app/lib/auth.ts index 009b8e3c..6dd6bdf2 100644 --- a/apps/rowboat/app/lib/auth.ts +++ b/apps/rowboat/app/lib/auth.ts @@ -1,13 +1,12 @@ import { z } from "zod"; -import { Claims } from "@auth0/nextjs-auth0"; import { ObjectId } from "mongodb"; import { usersCollection, projectsCollection, projectMembersCollection } from "./mongodb"; -import { getSession } from "@auth0/nextjs-auth0"; +import { auth0 } from "./auth0"; import { User, WithStringId } from "./types/types"; import { USE_AUTH } from "./feature_flags"; import { redirect } from "next/navigation"; -export const GUEST_SESSION: Claims = { +export const GUEST_SESSION = { email: "guest@rowboatlabs.com", email_verified: true, sub: "guest_user", @@ -39,9 +38,9 @@ export async function requireAuth(): Promise>> return GUEST_DB_USER; } - const { user } = await getSession() || {}; + const { user } = await auth0.getSession() || {}; if (!user) { - redirect('/api/auth/login'); + redirect('/auth/login'); } // fetch db user diff --git a/apps/rowboat/app/lib/auth0.ts b/apps/rowboat/app/lib/auth0.ts new file mode 100644 index 00000000..cefb6ae0 --- /dev/null +++ b/apps/rowboat/app/lib/auth0.ts @@ -0,0 +1,21 @@ +// lib/auth0.js + +import { Auth0Client } from "@auth0/nextjs-auth0/server"; + +// Initialize the Auth0 client +export const auth0 = new Auth0Client({ + // Options are loaded from environment variables by default + // Ensure necessary environment variables are properly set + domain: process.env.AUTH0_ISSUER_BASE_URL, + clientId: process.env.AUTH0_CLIENT_ID, + clientSecret: process.env.AUTH0_CLIENT_SECRET, + appBaseUrl: process.env.AUTH0_BASE_URL, + secret: process.env.AUTH0_SECRET, + + authorizationParameters: { + // In v4, the AUTH0_SCOPE and AUTH0_AUDIENCE environment variables for API authorized applications are no longer automatically picked up by the SDK. + // Instead, we need to provide the values explicitly. + scope: process.env.AUTH0_SCOPE, + audience: process.env.AUTH0_AUDIENCE, + } +}); \ No newline at end of file diff --git a/apps/rowboat/app/lib/billing.ts b/apps/rowboat/app/lib/billing.ts index f99e3400..dd7140ac 100644 --- a/apps/rowboat/app/lib/billing.ts +++ b/apps/rowboat/app/lib/billing.ts @@ -3,7 +3,6 @@ import { z } from 'zod'; import { Customer, AuthorizeRequest, AuthorizeResponse, LogUsageRequest, UsageResponse, CustomerPortalSessionResponse, PricesResponse, UpdateSubscriptionPlanRequest, UpdateSubscriptionPlanResponse, ModelsResponse } from './types/billing_types'; import { ObjectId } from 'mongodb'; import { projectsCollection, usersCollection } from './mongodb'; -import { getSession } from '@auth0/nextjs-auth0'; import { redirect } from 'next/navigation'; import { getUserFromSessionId, requireAuth } from './auth'; import { USE_BILLING } from './feature_flags'; diff --git a/apps/rowboat/app/lib/components/atmentions.ts b/apps/rowboat/app/lib/components/atmentions.ts index 8a17a39d..0d52adf1 100644 --- a/apps/rowboat/app/lib/components/atmentions.ts +++ b/apps/rowboat/app/lib/components/atmentions.ts @@ -23,6 +23,7 @@ export function createAtMentions({ agents, prompts, tools, currentAgentName }: C atMentions.push({ id, value: id, + label: `Agent: ${a.name}`, denotationChar: "@", // Add required properties for Match type link: id, target: "_self" @@ -35,6 +36,7 @@ export function createAtMentions({ agents, prompts, tools, currentAgentName }: C atMentions.push({ id, value: id, + label: `Prompt: ${prompt.name}`, denotationChar: "@", link: id, target: "_self" @@ -47,6 +49,7 @@ export function createAtMentions({ agents, prompts, tools, currentAgentName }: C atMentions.push({ id, value: id, + label: `Tool: ${tool.name}`, denotationChar: "@", link: id, target: "_self" diff --git a/apps/rowboat/app/lib/components/editable-field.tsx b/apps/rowboat/app/lib/components/editable-field.tsx index 9a91e5e3..4363e3f1 100644 --- a/apps/rowboat/app/lib/components/editable-field.tsx +++ b/apps/rowboat/app/lib/components/editable-field.tsx @@ -7,6 +7,8 @@ import { Label } from "./label"; import dynamic from "next/dynamic"; import { Match } from "./mentions_editor"; import { SparklesIcon } from "lucide-react"; +import { EntitySelectionContext } from "../../projects/[projectId]/workflow/workflow_editor"; +import { useContext } from "react"; const MentionsEditor = dynamic(() => import('./mentions_editor'), { ssr: false }); interface EditableFieldProps { @@ -30,6 +32,7 @@ interface EditableFieldProps { show: boolean; setShow: (show: boolean) => void; }; + onMentionNavigate?: (type: 'agent' | 'tool' | 'prompt', name: string) => void; } export function EditableField({ @@ -50,11 +53,15 @@ export function EditableField({ error, inline = false, showGenerateButton, + onMentionNavigate, }: EditableFieldProps) { const [isEditing, setIsEditing] = useState(false); const [localValue, setLocalValue] = useState(value); const ref = useRef(null); + // Use the context directly, will be undefined if not in provider + const entitySelection = useContext(EntitySelectionContext); + const validationResult = validate?.(localValue); const isValid = !validate || validationResult?.valid; @@ -73,6 +80,14 @@ export function EditableField({ setIsEditing(false); }); + const handleMentionNavigate = onMentionNavigate || ((type, name) => { + if (entitySelection) { + if (type === 'agent') entitySelection.onSelectAgent(name); + else if (type === 'tool') entitySelection.onSelectTool(name); + else if (type === 'prompt') entitySelection.onSelectPrompt(name); + } + }); + const commonProps = { autoFocus: true, value: localValue, @@ -178,19 +193,19 @@ export function EditableField({ {...commonProps} minRows={3} maxRows={20} - className="w-full" + className="w-full text-sm focus-visible:ring-0 focus:ring-0 outline-none" classNames={{ ...commonProps.classNames, - input: "rounded-md py-2", + input: "rounded-md py-2 text-base focus-visible:ring-0 focus:ring-0 outline-none", inputWrapper: "rounded-md border-medium py-1" }} />} {!multiline && {markdown &&
- +
} {!markdown &&
- +
} ) : ( diff --git a/apps/rowboat/app/lib/components/markdown-content.tsx b/apps/rowboat/app/lib/components/markdown-content.tsx index e59597ba..724f36b2 100644 --- a/apps/rowboat/app/lib/components/markdown-content.tsx +++ b/apps/rowboat/app/lib/components/markdown-content.tsx @@ -4,108 +4,138 @@ import { Match } from './mentions_editor'; export default function MarkdownContent({ content, - atValues = [] + atValues = [], + onMentionNavigate, }: { content: string; atValues?: Match[]; + onMentionNavigate?: (type: 'agent' | 'tool' | 'prompt', name: string) => void; }) { - return {children} - }, - h2({ children }) { - return

{children}

- }, - h3({ children }) { - return

{children}

- }, - h4({ children }) { - return

{children}

- }, - h5({ children }) { - return
{children}
- }, - h6({ children }) { - return
{children}
- }, - strong({ children }) { - return {children} - }, - p({ children }) { - return

{children}

- }, - ul({ children }) { - return
    {children}
- }, - ol({ children }) { - return
    {children}
- }, - table({ children }) { - return {children}
- }, - th({ children }) { - return {children} - }, - td({ children }) { - return {children} - }, - blockquote({ children }) { - return
{children}
; - }, - a(props) { - const { children, href, className, node, ...rest } = props; + return
+ {children} + }, + h2({ children }) { + return

{children}

+ }, + h3({ children }) { + return

{children}

+ }, + h4({ children }) { + return

{children}

+ }, + h5({ children }) { + return
{children}
+ }, + h6({ children }) { + return
{children}
+ }, + strong({ children }) { + return {children} + }, + p({ children }) { + return

{children}

+ }, + ul({ children }) { + return
    {children}
+ }, + ol({ children }) { + return
    {children}
+ }, + table({ children }) { + return {children}
+ }, + th({ children }) { + return {children} + }, + td({ children }) { + return {children} + }, + blockquote({ children }) { + return
{children}
; + }, + a(props) { + const { children, href, className, node, ...rest } = props; - // If this is a mention link, render it with mention styling - if (href === '#mention') { - let label: string = ''; - // Check if children is an array and get the first text element - if (Array.isArray(children) && children.length > 0) { - const text = children[0]; - if (typeof text === 'string') { - const parts = text.split('@'); + // If this is a mention link, render it with mention styling + if (href === '#mention') { + let label: string = ''; + // Check if children is an array and get the first text element + if (Array.isArray(children) && children.length > 0) { + const text = children[0]; + if (typeof text === 'string') { + const parts = text.split('@'); + if (parts.length === 2) { + label = parts[1]; + } + } + } else if (typeof children === 'string') { + // Fallback for direct string children + const parts = children.split('@'); if (parts.length === 2) { label = parts[1]; } } - } else if (typeof children === 'string') { - // Fallback for direct string children - const parts = children.split('@'); - if (parts.length === 2) { - label = parts[1]; - } - } - // check if the the mention is valid - const invalid = !atValues.some(atValue => atValue.id === label); - if (atValues.length > 0 && invalid) { + // Parse type and name for display + let displayLabel = label; + const typeMatch = label.match(/^(agent|tool|prompt):(.*)$/); + let type: 'agent' | 'tool' | 'prompt' | undefined; + let name: string | undefined; + if (typeMatch) { + type = typeMatch[1] as 'agent' | 'tool' | 'prompt'; + name = typeMatch[2]; + if (type === 'agent') displayLabel = `Agent: ${name}`; + else if (type === 'tool') displayLabel = `Tool: ${name}`; + else if (type === 'prompt') displayLabel = `Prompt: ${name}`; + } + + // check if the the mention is valid + const invalid = !atValues.some(atValue => atValue.id === label); + const handleMentionClick = (e: React.MouseEvent) => { + if (onMentionNavigate && type && name) { + e.preventDefault(); + onMentionNavigate(type, name); + } + }; + if (atValues.length > 0 && invalid) { + return ( + + {displayLabel} (!) + + ); + } return ( - - @{label} (!) + + {displayLabel} ); } - return ( - - @{label} - - ); - } - // Otherwise render normal link (your existing link component) - return - - {children} - - - - }, - }} - > - {content} - ; + // Otherwise render normal link (your existing link component) + return + + {children} + + + + }, + }} + > + {content} + +
; } \ No newline at end of file diff --git a/apps/rowboat/app/lib/components/mentions-editor.css b/apps/rowboat/app/lib/components/mentions-editor.css index 8a4eb524..c0e9f994 100644 --- a/apps/rowboat/app/lib/components/mentions-editor.css +++ b/apps/rowboat/app/lib/components/mentions-editor.css @@ -1,3 +1,5 @@ +@import "../../globals.css"; + /* Target both edit mode and view mode mentions */ .mention, .ql-editor p span[class*="bg-[#e"], /* Matches both #e8f2fe and #e0f2fe */ @@ -15,12 +17,12 @@ span[class*="bg-[#e"] { /* For view mode */ } /* Handle Next.js dark mode class if needed */ -:global(.dark) .mention, +/* :global(.dark) .mention, :global(.dark) .ql-editor p span[class*="bg-[#e"], :global(.dark) span[class*="bg-[#e"] { background-color: rgb(31 41 55) !important; color: rgb(243 244 246) !important; -} +} */ /* Override the inline styles */ .ql-editor p span[class*="bg-[#e0f2fe]"], diff --git a/apps/rowboat/app/lib/components/mentions_editor.tsx b/apps/rowboat/app/lib/components/mentions_editor.tsx index 54c80f53..e7eb3fd8 100644 --- a/apps/rowboat/app/lib/components/mentions_editor.tsx +++ b/apps/rowboat/app/lib/components/mentions_editor.tsx @@ -11,6 +11,7 @@ export type Match = { id: string; value: string; invalid?: boolean; + label?: string; [key: string]: string | boolean | undefined; }; @@ -18,7 +19,7 @@ class CustomMentionBlot extends MentionBlot { static render(data: any) { const element = document.createElement('span'); element.className = data.invalid ? 'invalid' : ''; - element.textContent = data.invalid ? `${data.value} (!)` : data.value; + element.textContent = data.invalid ? `${data.label || data.value} (!)` : (data.label || data.value); return element; } } @@ -154,7 +155,7 @@ export default function MentionEditor({ renderItem: (item: Match) => { const div = document.createElement('div'); div.className = "px-2 py-1 bg-white text-blue-800 hover:bg-blue-100 cursor-pointer"; - div.textContent = item.id; + div.textContent = item.label || item.id; return div; }, } diff --git a/apps/rowboat/app/lib/components/structured-list.tsx b/apps/rowboat/app/lib/components/structured-list.tsx index 5bb9e0f6..a816cb1f 100644 --- a/apps/rowboat/app/lib/components/structured-list.tsx +++ b/apps/rowboat/app/lib/components/structured-list.tsx @@ -26,7 +26,7 @@ export function ListItem({ onClick: () => void; disabled?: boolean; rightElement?: React.ReactNode; - selectedRef?: React.RefObject; + selectedRef?: React.RefObject; icon?: React.ReactNode; }) { return ( diff --git a/apps/rowboat/app/lib/components/user_button.tsx b/apps/rowboat/app/lib/components/user_button.tsx index ee3846ea..e612492f 100644 --- a/apps/rowboat/app/lib/components/user_button.tsx +++ b/apps/rowboat/app/lib/components/user_button.tsx @@ -1,5 +1,5 @@ 'use client'; -import { useUser } from '@auth0/nextjs-auth0/client'; +import { useUser } from '@auth0/nextjs-auth0'; import { Avatar, Dropdown, DropdownItem, DropdownSection, DropdownTrigger, DropdownMenu } from "@heroui/react"; import { useRouter } from 'next/navigation'; import Link from 'next/link'; @@ -24,7 +24,7 @@ export function UserButton({ useBilling }: { useBilling?: boolean }) { { if (key === 'logout') { - router.push('/api/auth/logout'); + router.push('/auth/logout'); } if (key === 'billing') { router.push('/billing'); diff --git a/apps/rowboat/app/lib/composio/composio.ts b/apps/rowboat/app/lib/composio/composio.ts new file mode 100644 index 00000000..3441f9fb --- /dev/null +++ b/apps/rowboat/app/lib/composio/composio.ts @@ -0,0 +1,410 @@ +import { z } from "zod"; +import { PrefixLogger } from "../utils"; + +const BASE_URL = 'https://backend.composio.dev/api/v3'; +const COMPOSIO_API_KEY = process.env.COMPOSIO_API_KEY || ""; + +export const ZAuthScheme = z.enum([ + 'API_KEY', + 'BASIC', + 'BASIC_WITH_JWT', + 'BEARER_TOKEN', + 'BILLCOM_AUTH', + 'CALCOM_AUTH', + 'COMPOSIO_LINK', + 'GOOGLE_SERVICE_ACCOUNT', + 'NO_AUTH', + 'OAUTH1', + 'OAUTH2', +]); + +export const ZConnectedAccountStatus = z.enum([ + 'INITIALIZING', + 'INITIATED', + 'ACTIVE', + 'FAILED', + 'EXPIRED', + 'INACTIVE', +]); + +export const ZToolkit = z.object({ + slug: z.string(), + name: z.string(), + meta: z.object({ + description: z.string(), + logo: z.string(), + tools_count: z.number(), + }), + no_auth: z.boolean(), + auth_schemes: z.array(ZAuthScheme), + composio_managed_auth_schemes: z.array(ZAuthScheme), +}); + +export const ZComposioField = z.object({ + name: z.string(), + displayName: z.string(), + type: z.string(), + description: z.string(), + required: z.boolean(), + default: z.string().nullable().optional(), +}); + +export const ZGetToolkitResponse = z.object({ + slug: z.string(), + name: z.string(), + composio_managed_auth_schemes: z.array(ZAuthScheme), + auth_config_details: z.array(z.object({ + name: z.string(), + mode: ZAuthScheme, + fields: z.object({ + auth_config_creation: z.object({ + required: z.array(ZComposioField), + optional: z.array(ZComposioField), + }), + connected_account_initiation: z.object({ + required: z.array(ZComposioField), + optional: z.array(ZComposioField), + }), + }) + })).nullable(), +}); + +export const ZTool = z.object({ + slug: z.string(), + name: z.string(), + description: z.string(), + toolkit: z.object({ + slug: z.string(), + name: z.string(), + logo: z.string(), + }), + input_parameters: z.object({ + type: z.literal('object'), + properties: z.record(z.string(), z.any()), + required: z.array(z.string()).optional(), + additionalProperties: z.boolean().optional(), + }), + no_auth: z.boolean(), +}); + +export const ZAuthConfig = z.object({ + id: z.string(), + is_composio_managed: z.boolean(), + auth_scheme: ZAuthScheme, +}); + +export const ZCredentials = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])); + +export const ZCreateAuthConfigRequest = z.object({ + toolkit: z.object({ + slug: z.string(), + }), + auth_config: z.discriminatedUnion('type', [ + z.object({ + type: z.literal('use_composio_managed_auth'), + name: z.string().optional(), + credentials: ZCredentials.optional(), + restrict_to_following_tools: z.array(z.string()).optional(), + }), + z.object({ + type: z.literal('use_custom_auth'), + authScheme: ZAuthScheme, + credentials: ZCredentials, + name: z.string().optional(), + proxy_config: z.object({ + proxy_url: z.string(), + proxy_auth_key: z.string().optional(), + }).optional(), + restrict_to_following_tools: z.array(z.string()).optional(), + }), + ]).optional(), +}); + +/* +{ + "toolkit": { + "slug": "github" + }, + "auth_config": { + "id": "ac_ZiLwFAWuGA7G", + "auth_scheme": "OAUTH2", + "is_composio_managed": false, + "restrict_to_following_tools": [] + } +} +*/ +export const ZCreateAuthConfigResponse = z.object({ + toolkit: z.object({ + slug: z.string(), + }), + auth_config: ZAuthConfig, +}); + +const ZConnectionData = z.object({ + authScheme: ZAuthScheme, + val: z.record(z.string(), z.unknown()) + .and(z.object({ + status: ZConnectedAccountStatus, + })), +}); + +export const ZCreateConnectedAccountRequest = z.object({ + auth_config: z.object({ + id: z.string(), + }), + connection: z.object({ + state: ZConnectionData.optional(), + user_id: z.string().optional(), + callback_url: z.string().optional(), + }), +}); + +/* +{ + "id": "ca_vTkCeLZSGab-", + "connectionData": { + "authScheme": "OAUTH2", + "val": { + "status": "INITIATED", + "code_verifier": "cd0103c5d8836a387adab1635b65ff0d2f51f77a1a79b7ff", + "redirectUrl": "https://backend.composio.dev/api/v3/s/DbTOWAyR", + "callback_url": "https://backend.composio.dev/api/v1/auth-apps/add" + } + }, + "status": "INITIATED", + "redirect_url": "https://backend.composio.dev/api/v3/s/DbTOWAyR", + "redirect_uri": "https://backend.composio.dev/api/v3/s/DbTOWAyR", + "deprecated": { + "uuid": "fe66d24b-59d8-4abf-adb2-d8f74353da9e", + "authConfigUuid": "8c4d4c84-56e2-4a80-aa59-9e84503381d8" + } +} +*/ +export const ZCreateConnectedAccountResponse = z.object({ + id: z.string(), + connectionData: ZConnectionData, +}); + +export const ZConnectedAccount = z.object({ + id: z.string(), + toolkit: z.object({ + slug: z.string(), + }), + auth_config: z.object({ + id: z.string(), + is_composio_managed: z.boolean(), + is_disabled: z.boolean(), + }), + status: ZConnectedAccountStatus, +}); + +const ZErrorResponse = z.object({ + error: z.object({ + message: z.string(), + error_code: z.number(), + suggested_fix: z.string().nullable(), + errors: z.array(z.string()).nullable(), + }), +}); + +export const ZError = z.object({ + error: z.enum([ + 'CUSTOM_OAUTH2_CONFIG_REQUIRED', + ]), +}); + +export const ZDeleteOperationResponse = z.object({ + success: z.boolean(), +}); + +export const ZListResponse = (schema: T) => z.object({ + items: z.array(schema), + next_cursor: z.string().nullable(), + total_pages: z.number(), + current_page: z.number(), + total_items: z.number(), +}); + +export async function composioApiCall( + schema: T, + url: string, + options: RequestInit = {}, +): Promise> { + const logger = new PrefixLogger('composioApiCall'); + logger.log(`[${options.method || 'GET'}] ${url}`, options); + + const then = Date.now(); + + try { + const response = await fetch(url, { + ...options, + headers: { + ...options.headers, + "x-api-key": COMPOSIO_API_KEY, + ...(options.method === 'POST' ? { + "Content-Type": "application/json", + } : {}), + }, + }); + const duration = Date.now() - then; + logger.log(`Took: ${duration}ms`); + const data = await response.json(); + if ('error' in data) { + const response = ZErrorResponse.parse(data); + throw new Error(`(code: ${response.error.error_code}): ${response.error.message}: ${response.error.suggested_fix}: ${response.error.errors?.join(', ')}`); + } + return schema.parse(data); + } catch (error) { + logger.log(`Error:`, error); + throw error; + } +} + +export async function listToolkits(cursor: string | null = null): Promise>>> { + const url = new URL(`${BASE_URL}/toolkits`); + + // set params + url.searchParams.set("sort_by", "usage"); + if (cursor) { + url.searchParams.set("cursor", cursor); + } + + // fetch + return composioApiCall(ZListResponse(ZToolkit), url.toString()); +} + +export async function getToolkit(toolkitSlug: string): Promise> { + const url = new URL(`${BASE_URL}/toolkits/${toolkitSlug}`); + return composioApiCall(ZGetToolkitResponse, url.toString()); +} + +export async function listTools(toolkitSlug: string, cursor: string | null = null): Promise>>> { + const url = new URL(`${BASE_URL}/tools`); + + // set params + url.searchParams.set("toolkit_slug", toolkitSlug); + if (cursor) { + url.searchParams.set("cursor", cursor); + } + + // fetch + return composioApiCall(ZListResponse(ZTool), url.toString()); +} + +export async function listAuthConfigs(toolkitSlug: string, cursor: string | null = null, managedOnly: boolean = false): Promise>>> { + const url = new URL(`${BASE_URL}/auth_configs`); + url.searchParams.set("toolkit_slug", toolkitSlug); + if (cursor) { + url.searchParams.set("cursor", cursor); + } + if (managedOnly) { + url.searchParams.set("is_composio_managed", "true"); + } + + // fetch + return composioApiCall(ZListResponse(ZAuthConfig), url.toString()); +} + +export async function createAuthConfig(request: z.infer): Promise> { + const url = new URL(`${BASE_URL}/auth_configs`); + return composioApiCall(ZCreateAuthConfigResponse, url.toString(), { + method: 'POST', + body: JSON.stringify(request), + }); +} + +export async function getAuthConfig(authConfigId: string): Promise> { + const url = new URL(`${BASE_URL}/auth_configs/${authConfigId}`); + return composioApiCall(ZAuthConfig, url.toString()); +} + +export async function deleteAuthConfig(authConfigId: string): Promise> { + const url = new URL(`${BASE_URL}/auth_configs/${authConfigId}`); + return composioApiCall(ZDeleteOperationResponse, url.toString(), { + method: 'DELETE', + }); +} + +// export async function createComposioManagedOauth2AuthConfig(toolkitSlug: string): Promise> { +// const response = await createAuthConfig({ +// toolkit: { +// slug: toolkitSlug, +// }, +// auth_config: { +// type: 'use_composio_managed_auth', +// }, +// }); +// return response.auth_config; +// } + +// export async function autocreateOauth2Integration(toolkitSlug: string): Promise> { +// // fetch toolkit +// const toolkit = await getToolkit(toolkitSlug); + +// // ensure oauth2 is supported +// if (!toolkit.auth_config_details?.some(config => config.mode === 'OAUTH2')) { +// throw new Error(`OAuth2 is not supported for toolkit ${toolkitSlug}`); +// } + +// // fetch existing auth configs +// const authConfigs = await fetchAuthConfigs(toolkitSlug); + +// // find a valid oauth2 config +// const oauth2AuthConfig = authConfigs.items.find(config => config.auth_scheme === 'OAUTH2'); + +// // if valid auth config, return it +// if (oauth2AuthConfig) { +// return oauth2AuthConfig; +// } + +// // check if composio managed oauth2 is supported +// if (toolkit.composio_managed_auth_schemes.includes('OAUTH2')) { +// return await createComposioManagedOauth2AuthConfig(toolkitSlug); +// } + +// // else return error +// return { +// error: 'CUSTOM_OAUTH2_CONFIG_REQUIRED', +// }; +// } + +export async function createConnectedAccount(request: z.infer): Promise> { + const url = new URL(`${BASE_URL}/connected_accounts`); + return composioApiCall(ZCreateConnectedAccountResponse, url.toString(), { + method: 'POST', + body: JSON.stringify(request), + }); +} + +// export async function createOauth2ConnectedAccount(toolkitSlug: string, userId: string, callbackUrl: string): Promise> { +// // fetch auth config +// const authConfig = await autocreateOauth2Integration(toolkitSlug); + +// // if error, return error +// if ('error' in authConfig) { +// return authConfig; +// } + +// // create connected account +// return await createConnectedAccount({ +// auth_config: { +// id: authConfig.id, +// }, +// connection: { +// user_id: userId, +// callback_url: callbackUrl, +// }, +// }); +// } + +export async function getConnectedAccount(connectedAccountId: string): Promise> { + const url = new URL(`${BASE_URL}/connected_accounts/${connectedAccountId}`); + return await composioApiCall(ZConnectedAccount, url.toString()); +} + +export async function deleteConnectedAccount(connectedAccountId: string): Promise> { + const url = new URL(`${BASE_URL}/connected_accounts/${connectedAccountId}`); + return await composioApiCall(ZDeleteOperationResponse, url.toString(), { + method: 'DELETE', + }); +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/constants/klavis.ts b/apps/rowboat/app/lib/constants/klavis.ts new file mode 100644 index 00000000..14a912f3 --- /dev/null +++ b/apps/rowboat/app/lib/constants/klavis.ts @@ -0,0 +1,31 @@ +// Server name to URL parameter mapping +export const SERVER_URL_PARAMS: Record = { + 'Google Calendar': 'gcalendar', + 'Google Drive': 'gdrive', + 'Google Docs': 'gdocs', + 'Google Sheets': 'gsheets', + 'Gmail': 'gmail', + 'GitHub': 'github', + 'Slack': 'slack', + 'Jira': 'jira', + 'Notion': 'notion', + 'Supabase': 'supabase', + 'WordPress': 'wordpress', + 'Asana': 'asana', + 'Close': 'close', + 'Confluence': 'confluence', + 'Salesforce': 'salesforce', + 'Linear': 'linear', + 'Attio': 'attio' +}; + +// Server name to environment variable mapping for client IDs +export const SERVER_CLIENT_ID_MAP: Record = { + 'GitHub': process.env.KLAVIS_GITHUB_CLIENT_ID, + 'Google Calendar': process.env.KLAVIS_GOOGLE_CLIENT_ID, + 'Google Drive': process.env.KLAVIS_GOOGLE_CLIENT_ID, + 'Google Docs': process.env.KLAVIS_GOOGLE_CLIENT_ID, + 'Google Sheets': process.env.KLAVIS_GOOGLE_CLIENT_ID, + 'Gmail': process.env.KLAVIS_GOOGLE_CLIENT_ID, + 'Slack': process.env.KLAVIS_SLACK_ID, +}; \ No newline at end of file diff --git a/apps/rowboat/app/lib/copilot/copilot.ts b/apps/rowboat/app/lib/copilot/copilot.ts new file mode 100644 index 00000000..879da5f9 --- /dev/null +++ b/apps/rowboat/app/lib/copilot/copilot.ts @@ -0,0 +1,201 @@ +import z from "zod"; +import { createOpenAI } from "@ai-sdk/openai"; +import { generateObject, streamText } from "ai"; +import { WithStringId } from "../types/types"; +import { Workflow } from "../types/workflow_types"; +import { CopilotChatContext, CopilotMessage } from "../types/copilot_types"; +import { DataSource } from "../types/datasource_types"; +import { PrefixLogger } from "../utils"; +import zodToJsonSchema from "zod-to-json-schema"; +import { COPILOT_INSTRUCTIONS_EDIT_AGENT } from "./copilot_edit_agent"; +import { COPILOT_INSTRUCTIONS_MULTI_AGENT } from "./copilot_multi_agent"; +import { COPILOT_MULTI_AGENT_EXAMPLE_1 } from "./example_multi_agent_1"; +import { CURRENT_WORKFLOW_PROMPT } from "./current_workflow"; + +const PROVIDER_API_KEY = process.env.PROVIDER_API_KEY || process.env.OPENAI_API_KEY || ''; +const PROVIDER_BASE_URL = process.env.PROVIDER_BASE_URL || undefined; +const COPILOT_MODEL = process.env.PROVIDER_COPILOT_MODEL || 'gpt-4.1'; +const AGENT_MODEL = process.env.PROVIDER_DEFAULT_MODEL || 'gpt-4.1'; + +const WORKFLOW_SCHEMA = JSON.stringify(zodToJsonSchema(Workflow)); + +const SYSTEM_PROMPT = [ + COPILOT_INSTRUCTIONS_MULTI_AGENT, + COPILOT_MULTI_AGENT_EXAMPLE_1, + CURRENT_WORKFLOW_PROMPT, +] + .join('\n\n') + .replace('{agent_model}', AGENT_MODEL) + .replace('{workflow_schema}', WORKFLOW_SCHEMA); + +const openai = createOpenAI({ + apiKey: PROVIDER_API_KEY, + baseURL: PROVIDER_BASE_URL, +}); + +const ZTextEvent = z.object({ + content: z.string(), +}); + +const ZDoneEvent = z.object({ + done: z.literal(true), +}); + +const ZEvent = z.union([ZTextEvent, ZDoneEvent]); + +function getContextPrompt(context: z.infer | null): string { + let prompt = ''; + switch (context?.type) { + case 'agent': + prompt = `**NOTE**:\nThe user is currently working on the following agent:\n${context.name}`; + break; + case 'tool': + prompt = `**NOTE**:\nThe user is currently working on the following tool:\n${context.name}`; + break; + case 'prompt': + prompt = `**NOTE**:The user is currently working on the following prompt:\n${context.name}`; + break; + case 'chat': + prompt = `**NOTE**: The user has just tested the following chat using the workflow above and has provided feedback / question below this json dump: +\`\`\`json +${JSON.stringify(context.messages)} +\`\`\` +`; + break; + } + return prompt; +} + +function getCurrentWorkflowPrompt(workflow: z.infer): string { + return `Context:\n\nThe current workflow config is: +\`\`\`json +${JSON.stringify(workflow)} +\`\`\` +`; +} + +function getDataSourcesPrompt(dataSources: WithStringId>[]): string { + let prompt = ''; + if (dataSources.length > 0) { + const simplifiedDataSources = dataSources.map(ds => ({ + id: ds._id, + name: ds.name, + description: ds.description, + data: ds.data, + })); + prompt = `**NOTE**: +The following data sources are available: +\`\`\`json +${JSON.stringify(simplifiedDataSources)} +\`\`\` +`; + } + return prompt; +} + +function updateLastUserMessage( + messages: z.infer[], + currentWorkflowPrompt: string, + contextPrompt: string, + dataSourcesPrompt: string = '', +): void { + const lastMessage = messages[messages.length - 1]; + if (lastMessage.role === 'user') { + lastMessage.content = `${currentWorkflowPrompt}\n\n${contextPrompt}\n\n${dataSourcesPrompt}\n\nUser: ${JSON.stringify(lastMessage.content)}`; + } +} + +export async function getEditAgentInstructionsResponse( + projectId: string, + context: z.infer | null, + messages: z.infer[], + workflow: z.infer, +): Promise { + const logger = new PrefixLogger('copilot /getUpdatedAgentInstructions'); + logger.log('context', context); + logger.log('projectId', projectId); + + // set the current workflow prompt + const currentWorkflowPrompt = getCurrentWorkflowPrompt(workflow); + + // set context prompt + let contextPrompt = getContextPrompt(context); + + // add the above prompts to the last user message + updateLastUserMessage(messages, currentWorkflowPrompt, contextPrompt); + + // call model + console.log("calling model", JSON.stringify({ + model: COPILOT_MODEL, + system: COPILOT_INSTRUCTIONS_EDIT_AGENT, + messages: messages, + })); + const { object } = await generateObject({ + model: openai(COPILOT_MODEL), + messages: [ + { + role: 'system', + content: SYSTEM_PROMPT, + }, + ...messages, + ], + schema: z.object({ + agent_instructions: z.string(), + }), + }); + + return object.agent_instructions; +} + +export async function* streamMultiAgentResponse( + projectId: string, + context: z.infer | null, + messages: z.infer[], + workflow: z.infer, + dataSources: WithStringId>[] +): AsyncIterable> { + const logger = new PrefixLogger('copilot /stream'); + logger.log('context', context); + logger.log('projectId', projectId); + + // set the current workflow prompt + const currentWorkflowPrompt = getCurrentWorkflowPrompt(workflow); + + // set context prompt + let contextPrompt = getContextPrompt(context); + + // set data sources prompt + let dataSourcesPrompt = getDataSourcesPrompt(dataSources); + + // add the above prompts to the last user message + updateLastUserMessage(messages, currentWorkflowPrompt, contextPrompt, dataSourcesPrompt); + + // call model + console.log("calling model", JSON.stringify({ + model: COPILOT_MODEL, + system: SYSTEM_PROMPT, + messages: messages, + })); + const { textStream } = streamText({ + model: openai(COPILOT_MODEL), + messages: [ + { + role: 'system', + content: SYSTEM_PROMPT, + }, + ...messages, + ], + }); + + // emit response chunks + for await (const chunk of textStream) { + yield { + content: chunk, + }; + } + + // done + yield { + done: true, + }; +} \ No newline at end of file diff --git a/apps/rowboat/app/lib/copilot/copilot_edit_agent.ts b/apps/rowboat/app/lib/copilot/copilot_edit_agent.ts new file mode 100644 index 00000000..ede593a4 --- /dev/null +++ b/apps/rowboat/app/lib/copilot/copilot_edit_agent.ts @@ -0,0 +1,65 @@ +export const COPILOT_INSTRUCTIONS_EDIT_AGENT = ` +## Role: +You are a copilot that helps the user create edit agent instructions. + +## Section 1 : Editing an Existing Agent + +When the user asks you to edit an existing agent, you should follow the steps below: + +1. Understand the user's request. +3. Retain as much of the original agent and only edit the parts that are relevant to the user's request. +3. If needed, ask clarifying questions to the user. Keep that to one turn and keep it minimal. +4. When you output an edited agent instructions, output the entire new agent instructions. + +## Section 8 : Creating New Agents + +When creating a new agent, strictly follow the format of this example agent. The user might not provide all information in the example agent, but you should still follow the format and add the missing information. + +example agent: +\`\`\` +## 🧑‍💼 Role: + +You are responsible for providing delivery information to the user. + +--- + +## ⚙️ Steps to Follow: + +1. Fetch the delivery details using the function: [@tool:get_shipping_details](#mention). +2. Answer the user's question based on the fetched delivery details. +3. If the user's issue concerns refunds or other topics beyond delivery, politely inform them that the information is not available within this chat and express regret for the inconvenience. + +--- +## 🎯 Scope: + +✅ In Scope: +- Questions about delivery status, shipping timelines, and delivery processes. +- Generic delivery/shipping-related questions where answers can be sourced from articles. + +❌ Out of Scope: +- Questions unrelated to delivery or shipping. +- Questions about products features, returns, subscriptions, or promotions. +- If a question is out of scope, politely inform the user and avoid providing an answer. + +--- + +## 📋 Guidelines: + +✔️ Dos: +- Use [@tool:get_shipping_details](#mention) to fetch accurate delivery information. +- Provide complete and clear answers based on the delivery details. +- For generic delivery questions, refer to relevant articles if necessary. +- Stick to factual information when answering. + +🚫 Don'ts: +- Do not provide answers without fetching delivery details when required. +- Do not leave the user with partial information. Refrain from phrases like 'please contact support'; instead, relay information limitations gracefully. +\`\`\` + +output format: +\`\`\`json +{ + "agent_instructions": "" +} +\`\`\` +`; \ No newline at end of file diff --git a/apps/rowboat/app/lib/copilot/copilot_multi_agent.ts b/apps/rowboat/app/lib/copilot/copilot_multi_agent.ts new file mode 100644 index 00000000..ddccc2b1 --- /dev/null +++ b/apps/rowboat/app/lib/copilot/copilot_multi_agent.ts @@ -0,0 +1,223 @@ +export const COPILOT_INSTRUCTIONS_MULTI_AGENT = ` +## Overview + +You are a helpful co-pilot for building and deploying multi-agent systems. Your goal is to perform tasks for the customer in designing a robust multi-agent system. You are allowed to ask one set of clarifying questions to the user. + +You can perform the following tasks: + +1. Create a multi-agent system +2. Create a new agent +3. Edit an existing agent +4. Improve an existing agent's instructions +5. Adding / editing / removing tools +6. Adding / editing / removing prompts + +If the user's request is not entirely clear, you can ask one turn of clarification. In the turn, you can ask up to 4 questions. Format the questions in a bulleted list. +### Out of Scope + +You are not equipped to perform the following tasks: + +1. Setting up RAG +2. Connecting tools to an API +3. Creating, editing or removing datasources +4. Creating, editing or removing projects +5. Creating, editing or removing Simulation scenarios + + +## Section 1 : Agent Behavior + +A agent can have one of the following behaviors: +1. Hub agent + primarily responsible for passing control to other agents connected to it. A hub agent's conversations with the user is limited to clarifying questions or simple small talk such as 'how can I help you today?', 'I'm good, how can I help you?' etc. A hub agent should not say that is is 'connecting you to an agent' and should just pass control to the agent. + +2. Info agent: + responsible for providing information and answering users questions. The agent usually gets its information through Retrieval Augmented Generation (RAG). An info agent usually performs an article look based on the user's question, answers the question and yields back control to the parent agent after its turn. + +3. Procedural agent : + responsible for following a set of steps such as the steps needed to complete a refund request. The steps might involve asking the user questions such as their email, calling functions such as get the user data, taking actions such as updating the user data. Procedures can contain nested if / else conditional statements. A single agent can typically follow up to 6 steps correctly. If the agent needs to follow more than 6 steps, decompose the agent into multiple smaller agents when creating new agents. + + +## Section 2 : Planning and Creating a Multi-Agent System + +When the user asks you to create agents for a multi agent system, you should follow the steps below: + +1. When necessary decompose the problem into multiple smaller agents. +2. Create a first draft of a new agent for each step in the plan. Use the format of the example agent. +3. Check if the agent needs any tools. Create any necessary tools and attach them to the agents. +4. If any part of the agent instruction seems common, create a prompt for it and attach it to the relevant agents. +5. Now ask the user for details for each agent, starting with the first agent. User Hub -> Info -> Procedural to prioritize which agent to ask for details first. +6. If there is an example agent, you should edit the example agent and rename it to create the hub agent. +7. Briefly list the assumptions you have made. + +## Section 3: Agent visibility and design patterns + +1. Agents can have 2 types of visibility - user_facing or internal. +2. Internal agents cannot put out messages to the user. Instead, their messages will be used by agents calling them (parent agents) to further compose their own responses. +3. User_facing agents can respond to the user directly +4. The start agent (main agent) should always have visbility set to user_facing. +5. You can use internal agents to create pipelines (Agent A calls Agent B calls Agent C, where Agent A is the only user_facing agent, which composes responses and talks to the user) by breaking up responsibilities across agents +6. A multi-agent system can be composed of internal and user_facing agents. If an agent needs to talk to the user, make it user_facing. If an agent has to purely carry out internal tasks (under the hood) then make it internal. You will typically use internal agents when a parent agent (user_facing) has complex tasks that need to be broken down into sub-agents (which will all be internal, child agents). +7. However, there are some important things you need to instruct the individual agents when they call other agents (you need to customize the below to the specific agent and its): + - SEQUENTIAL TRANSFERS AND RESPONSES: + A. BEFORE transferring to any agent: + - Plan your complete sequence of needed transfers + - Document which responses you need to collect + + B. DURING transfers: + - Transfer to only ONE agent at a time + - Wait for that agent's COMPLETE response and then proceed with the next agent + - Store the response for later use + - Only then proceed with the next transfer + - Never attempt parallel or simultaneous transfers + - CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output. + + C. AFTER receiving a response: + - Do not transfer to another agent until you've processed the current response + - If you need to transfer to another agent, wait for your current processing to complete + - Never transfer back to an agent that has already responded + + - COMPLETION REQUIREMENTS: + - Never provide final response until ALL required agents have been consulted + - Never attempt to get multiple responses in parallel + - If a transfer is rejected due to multiple handoffs: + A. Complete current response processing + B. Then retry the transfer as next in sequence + X. Continue until all required responses are collected + + - EXAMPLE: Suppose your instructions ask you to transfer to @agent:AgentA, @agent:AgentB and @agent:AgentC, first transfer to AgentA, wait for its response. Then transfer to AgentB, wait for its response. Then transfer to AgentC, wait for its response. Only after all 3 agents have responded, you should return the final response to the user. + +### When to make an agent user_facing and when to make it internal +- While the start agent (main agent) needs to be user_facing, it does **not** mean that **only** start agent (main agent) can be user_facing. Other agents can be user_facing as well if they need to communicate directly with the user. +- In general, you will use internal agents when they should carry out tasks and put out responses which should not be shown to the user. They can be used to create internal pipelines. For example, an interview analysis assistant might need to tell the user whether they passed the interview or not. However, under the hood, it can have several agents that read, rate and analyze the interview along different aspects. These will be internal agents. +- User_facing agents must be used when the agent has to talk to the user. For example, even though a credit card hub agent exists and is user_facing, you might want to make the credit card refunds agent user_facing if it is tasked with talking to the user about refunds and guiding them through the process. Its job is not purely under the hood and hence it has to be user_facing. +- The system works in such a way that every turn ends when a user_facing agent puts out a response, i.e., it is now the user's turn to respond back. However, internal agent responses do not end turns. Multiple internal agents can respond, which will all be used by a user_facing agent to respond to the user. + +## Section 4 : Editing an Existing Agent + +When the user asks you to edit an existing agent, you should follow the steps below: + +1. Understand the user's request. You can ask one set of clarifying questions if needed - keep it to at most 4 questions in a bulletted list. +2. Retain as much of the original agent and only edit the parts that are relevant to the user's request. +3. If needed, ask clarifying questions to the user. Keep that to one turn and keep it minimal. +4. When you output an edited agent instructions, output the entire new agent instructions. + +### Section 4.1 : Adding Examples to an Agent + +When adding examples to an agent use the below format for each example you create. Add examples to the example field in the agent config. Always add examples when creating a new agent, unless the user specifies otherwise. + +\`\`\` + - **User** : + - **Agent actions**: + - **Agent response**: " +\`\`\` + +Action involving calling other agents +1. If the action is calling another agent, denote it by 'Call [@agent:](#mention)' +2. If the action is calling another agent, don't include the agent response + +Action involving calling tools +1. If the action involves calling one or more tools, denote it by 'Call [@tool:tool_name_1](#mention), Call [@tool:tool_name_2](#mention) ... ' +2. If the action involves calling one or more tools, the corresponding response should have a placeholder to denote the output of tool call if necessary. e.g. 'Your order will be delivered on ' + +Style of Response +1. If there is a Style prompt or other prompts which mention how the agent should respond, use that as guide when creating the example response + +If the user doesn't specify how many examples, always add 5 examples. + +### Section 4.2 : Adding RAG data sources to an Agent + +When rag data sources are available you will be given the information on it like this: +\`\`\` +The following data sources are available: + +[{"id": "6822e76aa1358752955a455e", "name": "Handbook", "description": "This is a employee handbook", "active": true, "status": "ready", "error": null, "data": {"type": "text"}}] + +User: "can you add the handbook to the agent"] +\`\`\` + +You should use the name and description to understand the data source, and use the id to attach the data source to the agent. Example: + +'ragDataSources' = ["6822e76aa1358752955a455e"] + +Once you add the datasource ID to the agent, add a section to the agent instructions called RAG. Under that section, inform the agent that here are a set of data sources available to it and add the name and description of each attached data source. Instruct the agent to 'Call [@tool:rag_search](#mention) to pull information from any of the data sources before answering any questions on them'. + +Note: the rag_search tool searches across all data sources - it cannot call a specific data source. + +## Section 5 : Improving an Existing Agent + +When the user asks you to improve an existing agent, you should follow the steps below: + +1. Understand the user's request. +2. Go through the agents instructions line by line and check if any of the instrcution is underspecified. Come up with possible test cases. +3. Now look at each test case and edit the agent so that it has enough information to pass the test case. +4. If needed, ask clarifying questions to the user. Keep that to one turn and keep it minimal. + +## Section 6 : Adding / Editing / Removing Tools + +1. Follow the user's request and output the relevant actions and data based on the user's needs. +2. If you are removing a tool, make sure to remove it from all the agents that use it. +3. If you are adding a tool, make sure to add it to all the agents that need it. + +## Section 7 : Adding / Editing / Removing Prompts + +1. Follow the user's request and output the relevant actions and data based on the user's needs. +2. If you are removing a prompt, make sure to remove it from all the agents that use it. +3. If you are adding a prompt, make sure to add it to all the agents that need it. +4. Add all the fields for a new agent including a description, instructions, tools, prompts, etc. + +## Section 8 : Doing Multiple Actions at a Time + +1. you should present your changes in order of : tools, prompts, agents. +2. Make sure to add, remove tools and prompts from agents as required. + +## Section 9 : Creating New Agents + +When creating a new agent, strictly follow the format of this example agent. The user might not provide all information in the example agent, but you should still follow the format and add the missing information. + +example agent: +\`\`\` +## 🧑‍💼 Role:\nYou are the hub agent responsible for orchestrating the evaluation of interview transcripts between an executive search agency (Assistant) and a CxO candidate (User).\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the transcript in the specified format.\n2. FIRST: Send the transcript to [@agent:Evaluation Agent] for evaluation.\n3. Wait to receive the complete evaluation from the Evaluation Agent.\n4. THEN: Send the received evaluation to [@agent:Call Decision] to determine if the call quality is sufficient.\n5. Based on the Call Decision response:\n - If approved: Inform the user that the call has been approved and will proceed to profile creation.\n - If rejected: Inform the user that the call quality was insufficient and provide the reason.\n6. Return the final result (rejection reason or approval confirmation) to the user.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Orchestrating the sequential evaluation and decision process for interview transcripts.\n\n❌ Out of Scope:\n- Directly evaluating or creating profiles.\n- Handling transcripts not in the specified format.\n- Interacting with the individual evaluation agents.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Follow the strict sequence: Evaluation Agent first, then Call Decision.\n- Wait for each agent's complete response before proceeding.\n- Only interact with the user for final results or format clarification.\n\n🚫 Don'ts:\n- Do not perform evaluation or profile creation yourself.\n- Do not modify the transcript.\n- Do not try to get evaluations simultaneously.\n- Do not reference the individual evaluation agents.\n- CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output.\n\n# Examples\n- **User** : Here is the interview transcript: [2024-04-25, 10:00] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: \n 1. First call [@agent:Evaluation Agent](#mention)\n 2. Wait for complete evaluation\n 3. Then call [@agent:Call Decision](#mention)\n\n- **Agent receives evaluation and decision (approved)** :\n - **Agent response**: The call has been approved. Proceeding to candidate profile creation.\n\n- **Agent receives evaluation and decision (rejected)** :\n - **Agent response**: The call quality was insufficient to proceed. [Provide reason from Call Decision agent]\n\n- **User** : The transcript is in a different format.\n - **Agent response**: Please provide the transcript in the specified format: [,