Merge branch 'main' into main

Dockerfile.qdrant

@@ -0,0 +1,3 @@
+FROM qdrant/qdrant:latest
+
+RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/* 

README.md

@@ -22,15 +22,17 @@ Powered by OpenAI's Agents SDK, Rowboat is the fastest way to build multi-agents
    export OPENAI_API_KEY=your-openai-api-key
    ```
       
-2. Clone the repository and start Rowboat docker
+2. Clone the repository and start Rowboat
    ```bash
    git clone git@github.com:rowboatlabs/rowboat.git
    cd rowboat
-   docker-compose up --build
+   ./start.sh
    ```
 
 3. Access the app at [http://localhost:3000](http://localhost:3000).
 
+Note: We have added native RAG support including file-uploads and URL scraping. See the [RAG](https://docs.rowboatlabs.com/using_rag) section of our docs for this.
+
 Note: See the [Using custom LLM providers](https://docs.rowboatlabs.com/setup/#using-custom-llm-providers) section of our docs for using custom providers like OpenRouter and LiteLLM.
 
 ## Demo

apps/copilot/app.py

@@ -1,6 +1,6 @@
 from flask import Flask, request, jsonify, Response, stream_with_context
-from pydantic import BaseModel, ValidationError
-from typing import List
+from pydantic import BaseModel, ValidationError, Field
+from typing import List, Optional
 from copilot import UserMessage, AssistantMessage, get_response
 from streaming import get_streaming_response
 from lib import AgentContext, PromptContext, ToolContext, ChatContext
@@ -9,11 +9,24 @@ from functools import wraps
 from copilot import copilot_instructions_edit_agent
 import json
 
+class DataSource(BaseModel):
+    id: str = Field(alias='_id')
+    name: str
+    description: Optional[str] = None
+    active: bool = True
+    status: str  # 'pending' | 'ready' | 'error' | 'deleted'
+    error: Optional[str] = None
+    data: dict  # The discriminated union based on type
+
+    class Config:
+        populate_by_name = True
+
 class ApiRequest(BaseModel):
     messages: List[UserMessage | AssistantMessage]
     workflow_schema: str
     current_workflow_config: str
     context: AgentContext | PromptContext | ToolContext | ChatContext | None = None
+    dataSources: Optional[List[DataSource]] = None
 
 class ApiResponse(BaseModel):
     response: str
@@ -52,7 +65,10 @@ def health():
 @require_api_key
 def chat_stream():
     try:
-        request_data = ApiRequest(**request.json)
+        raw_data = request.json
+        print(f"Raw request JSON: {json.dumps(raw_data)}")
+        
+        request_data = ApiRequest(**raw_data)
         print(f"received /chat_stream request: {request_data}")
         validate_request(request_data)
 
@@ -61,7 +77,8 @@ def chat_stream():
                 messages=request_data.messages,
                 workflow_schema=request_data.workflow_schema,
                 current_workflow_config=request_data.current_workflow_config,
-                context=request_data.context
+                context=request_data.context,
+                dataSources=request_data.dataSources
             )
 
             for chunk in stream:

apps/copilot/copilot.py

@@ -1,7 +1,7 @@
 from openai import OpenAI
 from flask import Flask, request, jsonify
 from pydantic import BaseModel, ValidationError
-from typing import List, Dict, Any, Literal
+from typing import List, Dict, Any, Literal, Optional
 import json
 from lib import AgentContext, PromptContext, ToolContext, ChatContext
 from client import PROVIDER_COPILOT_MODEL
@@ -15,6 +15,15 @@ class AssistantMessage(BaseModel):
     role: Literal["assistant"]
     content: str
 
+class DataSource(BaseModel):
+    _id: str
+    name: str
+    description: Optional[str] = None
+    active: bool = True
+    status: str  # 'pending' | 'ready' | 'error' | 'deleted'
+    error: Optional[str] = None
+    data: dict  # The discriminated union based on type
+
 with open('copilot_edit_agent.md', 'r', encoding='utf-8') as file:
     copilot_instructions_edit_agent = file.read()
 
@@ -23,6 +32,7 @@ def get_response(
         workflow_schema: str,
         current_workflow_config: str,
         context: AgentContext | PromptContext | ToolContext | ChatContext | None = None,
+        dataSources: Optional[List[DataSource]] = None,
         copilot_instructions: str = copilot_instructions_edit_agent
     ) -> str:
     # if context is provided, create a prompt for the context
@@ -53,6 +63,16 @@ def get_response(
     else:
         context_prompt = ""
 
+    # Add dataSources to the context if provided
+    data_sources_prompt = ""
+    if dataSources:
+        data_sources_prompt = f"""
+**NOTE**: The following data sources are available:
+```json
+{json.dumps([ds.model_dump() for ds in dataSources])}
+```
+"""
+
     # add the workflow schema to the system prompt
     sys_prompt = copilot_instructions.replace("{workflow_schema}", workflow_schema)
 
@@ -66,6 +86,7 @@ The current workflow config is:
 ```
 
 {context_prompt}
+{data_sources_prompt}
 
 User: {last_message.content}
 """

apps/copilot/copilot_multi_agent.md

@@ -51,10 +51,10 @@ When the user asks you to create agents for a multi agent system, you should fol
 
 ## Section 3: Agent visibility and design patterns
 
-1. Agents can have 2 types of visibility - user_facing or internal. 
+1. Agents can have 2 types of visibility - user_facing or internal.
 2. Internal agents cannot put out messages to the user. Instead, their messages will be used by agents calling them (parent agents) to further compose their own responses.
 3. User_facing agents can respond to the user directly
-4. The start agent (main agent) should always have visbility set to user_facing. 
+4. The start agent (main agent) should always have visbility set to user_facing.
 5. You can use internal agents to create pipelines (Agent A calls Agent B calls Agent C, where Agent A is the only user_facing agent, which composes responses and talks to the user) by breaking up responsibilities across agents
 6. A multi-agent system can be composed of internal and user_facing agents. If an agent needs to talk to the user, make it user_facing. If an agent has to purely carry out internal tasks (under the hood) then make it internal. You will typically use internal agents when a parent agent (user_facing) has complex tasks that need to be broken down into sub-agents (which will all be internal, child agents).
 7. However, there are some important things you need to instruct the individual agents when they call other agents (you need to customize the below to the specific agent and its):
@@ -62,7 +62,7 @@ When the user asks you to create agents for a multi agent system, you should fol
     A. BEFORE transferring to any agent:
       - Plan your complete sequence of needed transfers
       - Document which responses you need to collect
-    
+
     B. DURING transfers:
       - Transfer to only ONE agent at a time
       - Wait for that agent's COMPLETE response and then proceed with the next agent
@@ -70,7 +70,7 @@ When the user asks you to create agents for a multi agent system, you should fol
       - Only then proceed with the next transfer
       - Never attempt parallel or simultaneous transfers
       - CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output.
-    
+
     C. AFTER receiving a response:
       - Do not transfer to another agent until you've processed the current response
       - If you need to transfer to another agent, wait for your current processing to complete
@@ -87,7 +87,7 @@ When the user asks you to create agents for a multi agent system, you should fol
   - EXAMPLE: Suppose your instructions ask you to transfer to @agent:AgentA, @agent:AgentB and @agent:AgentC, first transfer to AgentA, wait for its response. Then transfer to AgentB, wait for its response. Then transfer to AgentC, wait for its response. Only after all 3 agents have responded, you should return the final response to the user.
 
 ### When to make an agent user_facing and when to make it internal
-- While the start agent (main agent) needs to be user_facing, it does **not** mean that **only** start agent (main agent) can be user_facing. Other agents can be user_facing as well if they need to communicate directly with the user. 
+- While the start agent (main agent) needs to be user_facing, it does **not** mean that **only** start agent (main agent) can be user_facing. Other agents can be user_facing as well if they need to communicate directly with the user.
 - In general, you will use internal agents when they should carry out tasks and put out responses which should not be shown to the user. They can be used to create internal pipelines. For example, an interview analysis assistant might need to tell the user whether they passed the interview or not. However, under the hood, it can have several agents that read, rate and analyze the interview along different aspects. These will be internal agents.
 - User_facing agents must be used when the agent has to talk to the user. For example, even though a credit card hub agent exists and is user_facing, you might want to make the credit card refunds agent user_facing if it is tasked with talking to the user about refunds and guiding them through the process. Its job is not purely under the hood and hence it has to be user_facing.
 - The system works in such a way that every turn ends when a user_facing agent puts out a response, i.e., it is now the user's turn to respond back. However, internal agent responses do not end turns. Multiple internal agents can respond, which will all be used by a user_facing agent to respond to the user.
@@ -101,7 +101,7 @@ When the user asks you to edit an existing agent, you should follow the steps be
 3. If needed, ask clarifying questions to the user. Keep that to one turn and keep it minimal.
 4. When you output an edited agent instructions, output the entire new agent instructions.
 
-### Section 3.1 : Adding Examples to an Agent
+### Section 4.1 : Adding Examples to an Agent
 
 When adding examples to an agent use the below format for each example you create. Add examples to the example field in the agent config. Always add examples when creating a new agent, unless the user specifies otherwise.
 
@@ -124,6 +124,19 @@ Style of Response
 
 If the user doesn't specify how many examples, always add 5 examples.
 
+### Section 4.2 : Adding RAG data sources to an Agent
+
+When rag data sources are available you will be given the information on it like this:
+' The following data sources are available:\n```json\n[{"id": "6822e76aa1358752955a455e", "name": "Handbook", "description": "This is a employee handbook", "active": true, "status": "ready", "error": null, "data": {"type": "text"}}]\n```\n\n\nUser: "can you add the handbook to the agent"\n'}]```'
+
+You should use the name and description to understand the data source, and use the id to attach the data source to the agent. Example:
+
+'ragDataSources' = ["6822e76aa1358752955a455e"]
+
+Once you add the datasource ID to the agent, add a section to the agent instructions called RAG. Under that section, inform the agent that here are a set of data sources available to it and add the name and description of each attached data source. Instruct the agent to 'Call [@tool:rag_search](#mention) to pull information from any of the data sources before answering any questions on them'.
+
+Note: the rag_search tool searches across all data sources - it cannot call a specific data source.
+
 ## Section 5 : Improving an Existing Agent
 
 When the user asks you to improve an existing agent, you should follow the steps below:
@@ -158,7 +171,7 @@ When creating a new agent, strictly follow the format of this example agent. The
 example agent:
 ```
 ## 🧑‍💼 Role:\nYou are the hub agent responsible for orchestrating the evaluation of interview transcripts between an executive search agency (Assistant) and a CxO candidate (User).\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the transcript in the specified format.\n2. FIRST: Send the transcript to [@agent:Evaluation Agent] for evaluation.\n3. Wait to receive the complete evaluation from the Evaluation Agent.\n4. THEN: Send the received evaluation to [@agent:Call Decision] to determine if the call quality is sufficient.\n5. Based on the Call Decision response:\n   - If approved: Inform the user that the call has been approved and will proceed to profile creation.\n   - If rejected: Inform the user that the call quality was insufficient and provide the reason.\n6. Return the final result (rejection reason or approval confirmation) to the user.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Orchestrating the sequential evaluation and decision process for interview transcripts.\n\n❌ Out of Scope:\n- Directly evaluating or creating profiles.\n- Handling transcripts not in the specified format.\n- Interacting with the individual evaluation agents.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Follow the strict sequence: Evaluation Agent first, then Call Decision.\n- Wait for each agent's complete response before proceeding.\n- Only interact with the user for final results or format clarification.\n\n🚫 Don'ts:\n- Do not perform evaluation or profile creation yourself.\n- Do not modify the transcript.\n- Do not try to get evaluations simultaneously.\n- Do not reference the individual evaluation agents.\n- CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output.\n\n# Examples\n- **User** : Here is the interview transcript: [2024-04-25, 10:00] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: \n   1. First call [@agent:Evaluation Agent](#mention)\n   2. Wait for complete evaluation\n   3. Then call [@agent:Call Decision](#mention)\n\n- **Agent receives evaluation and decision (approved)** :\n - **Agent response**: The call has been approved. Proceeding to candidate profile creation.\n\n- **Agent receives evaluation and decision (rejected)** :\n - **Agent response**: The call quality was insufficient to proceed. [Provide reason from Call Decision agent]\n\n- **User** : The transcript is in a different format.\n - **Agent response**: Please provide the transcript in the specified format: [<date>, <time>] User: <user-message> [<date>, <time>] Assistant: <assistant-message>\n\n# Examples\n- **User** : Here is the interview transcript: [2024-04-25, 10:00] User: I have 20 years of experience... [2024-04-25, 10:01] Assistant: Can you describe your leadership style?\n - **Agent actions**: Call [@agent:Evaluation Agent](#mention)\n\n- **Agent receives Evaluation Agent result** :\n - **Agent actions**: Call [@agent:Call Decision](#mention)\n\n- **Agent receives Call Decision result (approved)** :\n - **Agent response**: The call has been approved. Proceeding to candidate profile creation.\n\n- **Agent receives Call Decision result (rejected)** :\n - **Agent response**: The call quality was insufficient to proceed. [Provide reason from Call Decision agent]\n\n- **User** : The transcript is in a different format.\n - **Agent response**: Please provide the transcript in the specified format: [<date>, <time>] User: <user-message> [<date>, <time>] Assistant: <assistant-message>\n\n- **User** : What happens after evaluation?\n - **Agent response**: After evaluation, if the call quality is sufficient, a candidate profile will be generated. Otherwise, you will receive feedback on why the call was rejected.
-'''
+```
 
 IMPORTANT: Use {agent_model} as the default model for new agents.
 
@@ -181,4 +194,23 @@ Note:
 
 If the user says 'Hi' or 'Hello', you should respond with a friendly greeting such as 'Hello! How can I help you today?'
 
-**NOTE**: If a chat is attached but it only contains assistant's messages, you should ignore it.
+**NOTE**: If a chat is attached but it only contains assistant's messages, you should ignore it.
+
+## Section 11 : In-product Support
+
+Below are FAQ's you should use when a use asks a questions on how to use the product (Rowboat).
+
+User Question : How do I connect an MCP server?
+Your Answer: Refer to https://docs.rowboatlabs.com/add_tools/ on how to connect MCP tools. Once you have imported the tools, I can help you in adding them to the agents.
+
+User Question : How do I connect an Webhook?
+Your Answer: Refer to https://docs.rowboatlabs.com/add_tools/ on how to connect a webhook. Once you have the tools setup, I can help you in adding them to the agents.
+
+User Question: How do I use the Rowboat API?
+Your Answer: Refer to https://docs.rowboatlabs.com/using_the_api/ on using the Rowboat API.
+
+User Question: How do I use the SDK?
+Your Answer: Refer to https://docs.rowboatlabs.com/using_the_sdk/ on using the Rowboat SDK.
+
+User Question: I want to add RAG?
+Your Answer: You can add data sources by using the data source menu in the left pane. You can fine more details in our docs: https://docs.rowboatlabs.com/using_rag.

apps/copilot/streaming.py

@@ -1,7 +1,7 @@
 from openai import OpenAI
 from flask import Flask, request, jsonify, Response, stream_with_context
-from pydantic import BaseModel, ValidationError
-from typing import List, Dict, Any, Literal
+from pydantic import BaseModel, ValidationError, Field
+from typing import List, Dict, Any, Literal, Optional
 import json
 from lib import AgentContext, PromptContext, ToolContext, ChatContext
 from client import PROVIDER_COPILOT_MODEL, PROVIDER_DEFAULT_MODEL
@@ -15,6 +15,18 @@ class AssistantMessage(BaseModel):
     role: Literal["assistant"]
     content: str
 
+class DataSource(BaseModel):
+    id: str = Field(alias='_id')
+    name: str
+    description: Optional[str] = None
+    active: bool = True
+    status: str  # 'pending' | 'ready' | 'error' | 'deleted'
+    error: Optional[str] = None
+    data: dict  # The discriminated union based on type
+
+    class Config:
+        populate_by_name = True
+
 with open('copilot_multi_agent.md', 'r', encoding='utf-8') as file:
     copilot_instructions_multi_agent = file.read()
 
@@ -39,6 +51,7 @@ def get_streaming_response(
         workflow_schema: str,
         current_workflow_config: str,
         context: AgentContext | PromptContext | ToolContext | ChatContext | None = None,
+        dataSources: Optional[List[DataSource]] = None,
 ) -> Any:
     # if context is provided, create a prompt for the context
     if context:
@@ -68,6 +81,20 @@ def get_streaming_response(
     else:
         context_prompt = ""
 
+    # Add dataSources to the context if provided
+    data_sources_prompt = ""
+    if dataSources:
+        print(f"Data sources found at project level: {dataSources}")
+        print(f"Data source IDs: {[ds.id for ds in dataSources]}")
+        data_sources_prompt = f"""
+**NOTE**: The following data sources are available:
+```json
+{json.dumps([ds.model_dump() for ds in dataSources])}
+```
+"""
+    else:
+        print("No data sources found at project level")
+
     # add the workflow schema to the system prompt
     sys_prompt = streaming_instructions.replace("{workflow_schema}", workflow_schema)
 
@@ -84,6 +111,7 @@ The current workflow config is:
 ```
 
 {context_prompt}
+{data_sources_prompt}
 
 User: {last_message.content}
 """
@@ -91,7 +119,7 @@ User: {last_message.content}
     updated_msgs = [{"role": "system", "content": sys_prompt}] + [
         message.model_dump() for message in messages
     ]
-
+    print(f"Input to copilot chat completions: {updated_msgs}")
     return completions_client.chat.completions.create(
         model=PROVIDER_COPILOT_MODEL,
         messages=updated_msgs,
@@ -113,6 +141,8 @@ def create_app():
             if not request_data or 'messages' not in request_data:
                 return jsonify({'error': 'No messages provided'}), 400
 
+            print(f"Raw request data: {request_data}")
+
             messages = [
                 UserMessage(**msg) if msg['role'] == 'user' else AssistantMessage(**msg)
                 for msg in request_data['messages']
@@ -121,13 +151,19 @@ def create_app():
             workflow_schema = request_data.get('workflow_schema', '')
             current_workflow_config = request_data.get('current_workflow_config', '')
             context = None  # You can add context handling if needed
+            dataSources = None
+            if 'dataSources' in request_data and request_data['dataSources']:
+                print(f"Raw dataSources from request: {request_data['dataSources']}")
+                dataSources = [DataSource(**ds) for ds in request_data['dataSources']]
+                print(f"Parsed dataSources: {dataSources}")
 
             def generate():
                 stream = get_streaming_response(
                     messages=messages,
                     workflow_schema=workflow_schema,
                     current_workflow_config=current_workflow_config,
-                    context=context
+                    context=context,
+                    dataSources=dataSources
                 )
 
                 for chunk in stream:

apps/docs/docs/using_rag.md

@@ -0,0 +1,104 @@
+# Using RAG in Rowboat
+
+Rowboat provides multiple ways to enhance your agents with Retrieval-Augmented Generation (RAG). This guide will help you set up and use each RAG feature.
+
+## Quick Start
+
+Text RAG and local file uploads are enabled by default - no configuration needed! Just start using them right away.
+
+## Available RAG Features
+
+### 1. Text RAG
+✅ Enabled by default:
+
+- Process and reason over text content directly
+- No configuration required
+
+### 2. Local File Uploads
+✅ Enabled by default:
+
+- Upload PDF files directly from your device
+- Files are stored locally
+- No configuration required
+- Files are parsed using OpenAI by default
+
+### 3. S3 File Uploads
+To enable S3 file uploads, set the following variables:
+
+```bash
+# Enable S3 uploads
+export USE_RAG_S3_UPLOADS=true
+
+# S3 Configuration
+export AWS_ACCESS_KEY_ID=your_access_key
+export AWS_SECRET_ACCESS_KEY=your_secret_key
+export RAG_UPLOADS_S3_BUCKET=your_bucket_name
+export RAG_UPLOADS_S3_REGION=your_region
+```
+
+### 4. URL Scraping
+To enable URL scraping, set the following variables:
+
+```bash
+# Enable URL scraping
+export USE_RAG_SCRAPING=true
+
+# Firecrawl API key for web scraping
+export FIRECRAWL_API_KEY=your_firecrawl_api_key
+```
+
+## File Parsing Options
+
+### Default Parsing (OpenAI)
+By default, uploaded PDF files are parsed using `gpt-4o`. You can customize this by setting the following:
+
+```bash
+# Override the default parsing model
+export FILE_PARSING_MODEL=your-preferred-model
+```
+
+You can also change the model provider like so:
+```bash
+# Optional: Override the parsing provider settings
+export FILE_PARSING_PROVIDER_BASE_URL=your-provider-base-url
+export FILE_PARSING_PROVIDER_API_KEY=your-provider-api-key
+```
+
+### Using Gemini for File Parsing
+To use Google's Gemini model for parsing uploaded PDFs, set the following variable:
+
+```bash
+# Enable Gemini for file parsing
+export USE_GEMINI_FILE_PARSING=true
+export GOOGLE_API_KEY=your_google_api_key
+```
+
+## Embedding Model options
+
+By default, Rowboat uses OpenAI's `text-embedding-3-small` model for generating embeddings. You can customize this by setting the following:
+
+```bash
+# Override the default embedding model
+export EMBEDDING_MODEL=your-preferred-model
+export EMBEDDING_VECTOR_SIZE=1536
+```
+
+**Important NOTE**
+
+The default size for the vectors index is 1536. If you change this value, then you must delete the index and set it up again:
+```bash
+docker-compose --profile delete_qdrant --profile qdrant up --build delete_qdrant qdrant
+```
+followed by:
+```bash
+./start # this will recreate the index
+```
+
+You can also change the model provider like so:
+```bash
+# Optional: Override the embedding provider settings
+export EMBEDDING_PROVIDER_BASE_URL=your-provider-base-url
+export EMBEDDING_PROVIDER_API_KEY=your-provider-api-key
+```
+
+If you don't specify the provider settings, Rowboat will use OpenAI as the default provider.

apps/docs/mkdocs.yml

@@ -14,6 +14,7 @@ nav:
     - Test chats in the playground: playground.md
     - Add tools: add_tools.md
     - Update agents: update_agents.md
+    - Using RAG: using_rag.md
 
   - API & SDK:
     - Using the API: using_the_api.md

apps/rowboat/app/actions/copilot_actions.ts

@@ -2,10 +2,12 @@
 import { 
     convertToCopilotWorkflow, convertToCopilotMessage, convertToCopilotApiMessage,
     convertToCopilotApiChatContext, CopilotAPIResponse, CopilotAPIRequest,
-    CopilotChatContext, CopilotMessage, CopilotAssistantMessage, CopilotWorkflow 
+    CopilotChatContext, CopilotMessage, CopilotAssistantMessage, CopilotWorkflow,
+    CopilotDataSource
 } from "../lib/types/copilot_types";
 import { 
     Workflow} from "../lib/types/workflow_types";
+import { DataSource } from "../lib/types/datasource_types";
 import { z } from 'zod';
 import { zodToJsonSchema } from 'zod-to-json-schema';
 import { assert } from "node:console";
@@ -18,7 +20,8 @@ export async function getCopilotResponse(
     projectId: string,
     messages: z.infer<typeof CopilotMessage>[],
     current_workflow_config: z.infer<typeof Workflow>,
-    context: z.infer<typeof CopilotChatContext> | null
+    context: z.infer<typeof CopilotChatContext> | null,
+    dataSources?: z.infer<typeof DataSource>[]
 ): Promise<{
     message: z.infer<typeof CopilotAssistantMessage>;
     rawRequest: unknown;
@@ -35,6 +38,24 @@ export async function getCopilotResponse(
         workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)),
         current_workflow_config: JSON.stringify(convertToCopilotWorkflow(current_workflow_config)),
         context: context ? convertToCopilotApiChatContext(context) : null,
+        dataSources: dataSources ? dataSources.map(ds => {
+            console.log('Original data source:', JSON.stringify(ds));
+            // First parse to validate, then ensure _id is included
+            CopilotDataSource.parse(ds); // validate but don't use the result
+            // Cast to any to handle the WithStringId type
+            const withId = ds as any;
+            const result = {
+                _id: withId._id,
+                name: withId.name,
+                description: withId.description,
+                active: withId.active,
+                status: withId.status,
+                error: withId.error,
+                data: withId.data
+            };
+            console.log('Processed data source:', JSON.stringify(result));
+            return result;
+        }) : undefined,
     };
     console.log(`sending copilot request`, JSON.stringify(request));
 
@@ -96,7 +117,8 @@ export async function getCopilotResponseStream(
     projectId: string,
     messages: z.infer<typeof CopilotMessage>[],
     current_workflow_config: z.infer<typeof Workflow>,
-    context: z.infer<typeof CopilotChatContext> | null
+    context: z.infer<typeof CopilotChatContext> | null,
+    dataSources?: z.infer<typeof DataSource>[]
 ): Promise<{
     streamId: string;
 }> {
@@ -111,6 +133,7 @@ export async function getCopilotResponseStream(
         workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)),
         current_workflow_config: JSON.stringify(convertToCopilotWorkflow(current_workflow_config)),
         context: context ? convertToCopilotApiChatContext(context) : null,
+        dataSources: dataSources ? dataSources.map(ds => CopilotDataSource.parse(ds)) : undefined,
     };
 
     // serialize the request

apps/rowboat/app/actions/datasource_actions.ts

@@ -10,6 +10,7 @@ import { WithStringId } from "../lib/types/types";
 import { DataSourceDoc } from "../lib/types/datasource_types";
 import { DataSource } from "../lib/types/datasource_types";
 import { uploadsS3Client } from "../lib/uploads_s3_client";
+import { USE_RAG_S3_UPLOADS } from "../lib/feature_flags";
 
 export async function getDataSource(projectId: string, sourceId: string): Promise<WithStringId<z.infer<typeof DataSource>>> {
     await projectAuthCheck(projectId);
@@ -42,11 +43,13 @@ export async function listDataSources(projectId: string): Promise<WithStringId<z
 export async function createDataSource({
     projectId,
     name,
+    description,
     data,
     status = 'pending',
 }: {
     projectId: string,
     name: string,
+    description?: string,
     data: z.infer<typeof DataSource>['data'],
     status?: 'pending' | 'ready',
 }): Promise<WithStringId<z.infer<typeof DataSource>>> {
@@ -56,12 +59,18 @@ export async function createDataSource({
         projectId: projectId,
         active: true,
         name: name,
+        description,
         createdAt: (new Date()).toISOString(),
         attempts: 0,
-        status: status,
         version: 1,
         data,
     };
+
+    // Only set status for non-file data sources
+    if (data.type !== 'files_local' && data.type !== 'files_s3') {
+        source.status = status;
+    }
+
     await dataSourcesCollection.insertOne(source);
 
     const { _id, ...rest } = source as WithId<z.infer<typeof DataSource>>;
@@ -154,7 +163,7 @@ export async function addDocsToDataSource({
     }[]
 }): Promise<void> {
     await projectAuthCheck(projectId);
-    await getDataSource(projectId, sourceId);
+    const source = await getDataSource(projectId, sourceId);
 
     await dataSourceDocsCollection.insertMany(docData.map(doc => {
         const record: z.infer<typeof DataSourceDoc> = {
@@ -173,19 +182,22 @@ export async function addDocsToDataSource({
         return recordWithId;
     }));
 
-    await dataSourcesCollection.updateOne(
-        { _id: new ObjectId(sourceId) },
-        {
-            $set: {
-                status: 'pending',
-                attempts: 0,
-                lastUpdatedAt: new Date().toISOString(),
-            },
-            $inc: {
-                version: 1,
-            },
-        }
-    );
+    // Only set status to pending when files are added
+    if (docData.length > 0 && (source.data.type === 'files_local' || source.data.type === 'files_s3')) {
+        await dataSourcesCollection.updateOne(
+            { _id: new ObjectId(sourceId) },
+            {
+                $set: {
+                    status: 'pending',
+                    attempts: 0,
+                    lastUpdatedAt: new Date().toISOString(),
+                },
+                $inc: {
+                    version: 1,
+                },
+            }
+        );
+    }
 }
 
 export async function listDocsInDataSource({
@@ -279,26 +291,27 @@ export async function getDownloadUrlForFile(
 ): Promise<string> {
     await projectAuthCheck(projectId);
     await getDataSource(projectId, sourceId);
-
-    // fetch s3 key for file
     const file = await dataSourceDocsCollection.findOne({
         sourceId,
         _id: new ObjectId(fileId),
-        'data.type': 'file',
+        'data.type': { $in: ['file_local', 'file_s3'] },
     });
     if (!file) {
         throw new Error('File not found');
     }
-    if (file.data.type !== 'file') {
-        throw new Error('File not found');
+
+    // if local, return path
+    if (file.data.type === 'file_local') {
+        return `/api/uploads/${fileId}`;
+    } else if (file.data.type === 'file_s3') {
+        const command = new GetObjectCommand({
+            Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
+            Key: file.data.s3Key,
+        });
+        return await getSignedUrl(uploadsS3Client, command, { expiresIn: 60 }); // URL valid for 1 minute
     }
 
-    const command = new GetObjectCommand({
-        Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
-        Key: file.data.s3Key,
-    });
-
-    return await getSignedUrl(uploadsS3Client, command, { expiresIn: 60 }); // URL valid for 1 minute
+    throw new Error('Invalid file type');
 }
 
 export async function getUploadUrlsForFilesDataSource(
@@ -307,38 +320,73 @@ export async function getUploadUrlsForFilesDataSource(
     files: { name: string; type: string; size: number }[]
 ): Promise<{
     fileId: string,
-    presignedUrl: string,
-    s3Key: string,
+    uploadUrl: string,
+    path: string,
 }[]> {
     await projectAuthCheck(projectId);
     const source = await getDataSource(projectId, sourceId);
-    if (source.data.type !== 'files') {
+    if (source.data.type !== 'files_local' && source.data.type !== 'files_s3') {
         throw new Error('Invalid files data source');
     }
 
     const urls: {
         fileId: string,
-        presignedUrl: string,
-        s3Key: string,
+        uploadUrl: string,
+        path: string,
     }[] = [];
 
     for (const file of files) {
         const fileId = new ObjectId().toString();
-        const projectIdPrefix = projectId.slice(0, 2); // 2 characters from the start of the projectId
-        const s3Key = `datasources/files/${projectIdPrefix}/${projectId}/${sourceId}/${fileId}/${file.name}`;
-        // Generate presigned URL
-        const command = new PutObjectCommand({
-            Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
-            Key: s3Key,
-            ContentType: file.type,
-        });
-        const presignedUrl = await getSignedUrl(uploadsS3Client, command, { expiresIn: 10 * 60 }); // valid for 10 minutes
-        urls.push({
-            fileId,
-            presignedUrl,
-            s3Key,
-        });
+
+        if (source.data.type === 'files_s3') {
+            // Generate presigned URL
+            const projectIdPrefix = projectId.slice(0, 2); // 2 characters from the start of the projectId
+            const path = `datasources/files/${projectIdPrefix}/${projectId}/${sourceId}/${fileId}/${file.name}`;
+            const command = new PutObjectCommand({
+                Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
+                Key: path,
+                ContentType: file.type,
+            });
+            const uploadUrl = await getSignedUrl(uploadsS3Client, command, { expiresIn: 10 * 60 }); // valid for 10 minutes
+            urls.push({
+                fileId,
+                uploadUrl,
+                path,
+            });
+        } else if (source.data.type === 'files_local') {
+            // Generate local upload URL
+            urls.push({
+                fileId,
+                uploadUrl: '/api/uploads/' + fileId,
+                path: '/api/uploads/' + fileId,
+            });
+        }
     }
 
     return urls;
 }
+
+export async function updateDataSource({
+    projectId,
+    sourceId,
+    description,
+}: {
+    projectId: string,
+    sourceId: string,
+    description: string,
+}) {
+    await projectAuthCheck(projectId);
+    await getDataSource(projectId, sourceId);
+
+    await dataSourcesCollection.updateOne({
+        _id: new ObjectId(sourceId),
+    }, {
+        $set: {
+            description,
+            lastUpdatedAt: (new Date()).toISOString(),
+        },
+        $inc: {
+            version: 1,
+        },
+    });
+}

apps/rowboat/app/api/uploads/[fileId]/route.ts

@@ -0,0 +1,87 @@
+import { NextRequest, NextResponse } from 'next/server';
+import path from 'path';
+import fs from 'fs/promises';
+import fsSync from 'fs';
+import { dataSourceDocsCollection } from '@/app/lib/mongodb';
+import { ObjectId } from 'mongodb';
+
+const UPLOADS_DIR = process.env.RAG_UPLOADS_DIR || '/uploads';
+
+// PUT endpoint to handle file uploads
+export async function PUT(
+    request: NextRequest,
+    { params }: { params: { fileId: string } }
+) {
+    const fileId = params.fileId;
+    if (!fileId) {
+        return NextResponse.json({ error: 'Missing file ID' }, { status: 400 });
+    }
+
+    const filePath = path.join(UPLOADS_DIR, fileId);
+
+    try {
+        const data = await request.arrayBuffer();
+        await fs.writeFile(filePath, new Uint8Array(data));
+        
+        return NextResponse.json({ success: true });
+    } catch (error) {
+        console.error('Error saving file:', error);
+        return NextResponse.json(
+            { error: 'Failed to save file' },
+            { status: 500 }
+        );
+    }
+}
+
+// GET endpoint to handle file downloads
+export async function GET(
+    request: NextRequest,
+    { params }: { params: { fileId: string } }
+) {
+    const fileId = params.fileId;
+    if (!fileId) {
+        return NextResponse.json({ error: 'Missing file ID' }, { status: 400 });
+    }
+
+    const filePath = path.join(UPLOADS_DIR, fileId);
+
+    // get mimetype from database
+    const doc = await dataSourceDocsCollection.findOne({ _id: new ObjectId(fileId) });
+    if (!doc) {
+        return NextResponse.json({ error: 'File not found' }, { status: 404 });
+    }
+
+    if (doc.data.type !== 'file_local') {
+        return NextResponse.json({ error: 'File is not local' }, { status: 400 });
+    }
+    const mimeType = 'application/octet-stream';
+    const fileName = doc.data.name;
+
+    try {
+        // Check if file exists
+        await fs.access(filePath);
+        // Create a readable stream
+        const nodeStream = fsSync.createReadStream(filePath);
+        // Convert Node.js stream to Web stream
+        const webStream = new ReadableStream({
+            start(controller) {
+                nodeStream.on('data', (chunk) => controller.enqueue(chunk));
+                nodeStream.on('end', () => controller.close());
+                nodeStream.on('error', (err) => controller.error(err));
+            }
+        });
+        return new NextResponse(webStream, {
+            status: 200,
+            headers: {
+                'Content-Type': mimeType,
+                'Content-Disposition': `attachment; filename="${fileName}"`,
+            },
+        });
+    } catch (error) {
+        console.error('Error reading file:', error);
+        return NextResponse.json(
+            { error: 'File not found' },
+            { status: 404 }
+        );
+    }
+}

apps/rowboat/app/lib/embedding.ts

@@ -1,3 +1,12 @@
-import { openai } from "@ai-sdk/openai";
+import { createOpenAI } from "@ai-sdk/openai";
 
-export const embeddingModel = openai.embedding('text-embedding-3-small');
+const EMBEDDING_PROVIDER_API_KEY = process.env.EMBEDDING_PROVIDER_API_KEY || process.env.OPENAI_API_KEY || '';
+const EMBEDDING_PROVIDER_BASE_URL = process.env.EMBEDDING_PROVIDER_BASE_URL || undefined;
+const EMBEDDING_MODEL = process.env.EMBEDDING_MODEL || 'text-embedding-3-small';
+
+const openai = createOpenAI({
+    apiKey: EMBEDDING_PROVIDER_API_KEY,
+    baseURL: EMBEDDING_PROVIDER_BASE_URL,
+});
+
+export const embeddingModel = openai.embedding(EMBEDDING_MODEL);

apps/rowboat/app/lib/feature_flags.ts

@@ -3,6 +3,8 @@ export const USE_RAG_UPLOADS = process.env.USE_RAG_UPLOADS === 'true';
 export const USE_RAG_SCRAPING = process.env.USE_RAG_SCRAPING === 'true';
 export const USE_CHAT_WIDGET = process.env.USE_CHAT_WIDGET === 'true';
 export const USE_AUTH = process.env.USE_AUTH === 'true';
+export const USE_RAG_S3_UPLOADS = process.env.USE_RAG_S3_UPLOADS === 'true';
+export const USE_GEMINI_FILE_PARSING = process.env.USE_GEMINI_FILE_PARSING === 'true';
 
 // Hardcoded flags
 export const USE_MULTIPLE_PROJECTS = true;

apps/rowboat/app/lib/project_templates.ts

@@ -33,8 +33,26 @@ export const templates: { [key: string]: z.infer<typeof WorkflowTemplate> } = {
                     "properties": {},
                 },
                 "isLibrary": true
+            },
+            {
+                "name": "rag_search",
+                "description": "Fetch articles with knowledge relevant to the query",
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "query": {
+                            "type": "string",
+                            "description": "The query to retrieve articles for"
+                        }
+                    },
+                    "required": [
+                        "query"
+                    ]
+                },
+                "isLibrary": true
             }
         ],
+        
     }
 }
 

apps/rowboat/app/lib/types/copilot_types.ts

@@ -3,11 +3,42 @@ import { Workflow } from "./workflow_types";
 import { apiV1 } from "rowboat-shared"
 import { AgenticAPIChatMessage } from "./agents_api_types";
 import { convertToAgenticAPIChatMessages } from "./agents_api_types";
+import { DataSource } from "./datasource_types";
+
+// Create a filtered version of DataSource for copilot
+export const CopilotDataSource = z.object({
+    _id: z.string(),
+    name: z.string(),
+    description: z.string().optional(),
+    active: z.boolean().default(true),
+    status: z.union([
+        z.literal('pending'),
+        z.literal('ready'),
+        z.literal('error'),
+        z.literal('deleted'),
+    ]),
+    error: z.string().optional(),
+    data: z.discriminatedUnion('type', [
+        z.object({
+            type: z.literal('urls'),
+        }),
+        z.object({
+            type: z.literal('files_local'),
+        }),
+        z.object({
+            type: z.literal('files_s3'),
+        }),
+        z.object({
+            type: z.literal('text'),
+        })
+    ]),
+}).passthrough();
 
 export const CopilotWorkflow = Workflow.omit({
     lastUpdatedAt: true,
     projectId: true,
-});export const CopilotUserMessage = z.object({
+});
+export const CopilotUserMessage = z.object({
     role: z.literal('user'),
     content: z.string(),
 });
@@ -77,6 +108,7 @@ export const CopilotAPIRequest = z.object({
     workflow_schema: z.string(),
     current_workflow_config: z.string(),
     context: CopilotApiChatContext.nullable(),
+    dataSources: z.array(CopilotDataSource).optional(),
 });
 export const CopilotAPIResponse = z.union([
     z.object({

apps/rowboat/app/lib/types/datasource_types.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 
 export const DataSource = z.object({
     name: z.string(),
+    description: z.string().optional(),
     projectId: z.string(),
     active: z.boolean().default(true),
     status: z.union([
@@ -9,7 +10,7 @@ export const DataSource = z.object({
         z.literal('ready'),
         z.literal('error'),
         z.literal('deleted'),
-    ]),
+    ]).optional(),
     version: z.number(),
     error: z.string().optional(),
     createdAt: z.string().datetime(),
@@ -22,7 +23,10 @@ export const DataSource = z.object({
             type: z.literal('urls'),
         }),
         z.object({
-            type: z.literal('files'),
+            type: z.literal('files_local'),
+        }),
+        z.object({
+            type: z.literal('files_s3'),
         }),
         z.object({
             type: z.literal('text'),
@@ -50,7 +54,13 @@ export const DataSourceDoc = z.object({
             url: z.string(),
         }),
         z.object({
-            type: z.literal('file'),
+            type: z.literal('file_local'),
+            name: z.string(),
+            size: z.number(),
+            mimeType: z.string(),
+        }),
+        z.object({
+            type: z.literal('file_s3'),
             name: z.string(),
             size: z.number(),
             mimeType: z.string(),

apps/rowboat/app/projects/[projectId]/copilot/app.tsx

@@ -5,6 +5,7 @@ import { useRef, useState, createContext, useContext, useCallback, forwardRef, u
 import { CopilotChatContext } from "../../../lib/types/copilot_types";
 import { CopilotMessage } from "../../../lib/types/copilot_types";
 import { Workflow } from "@/app/lib/types/workflow_types";
+import { DataSource } from "@/app/lib/types/datasource_types";
 import { z } from "zod";
 import { Action as WorkflowDispatch } from "../workflow/workflow_editor";
 import { Panel } from "@/components/common/panel-common";
@@ -30,9 +31,10 @@ interface AppProps {
     onCopyJson?: (data: { messages: any[] }) => void;
     onMessagesChange?: (messages: z.infer<typeof CopilotMessage>[]) => void;
     isInitialState?: boolean;
+    dataSources?: z.infer<typeof DataSource>[];
 }
 
-const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({
+const App = forwardRef<{ handleCopyChat: () => void; handleUserMessage: (message: string) => void }, AppProps>(function App({
     projectId,
     workflow,
     dispatch,
@@ -40,6 +42,7 @@ const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({
     onCopyJson,
     onMessagesChange,
     isInitialState = false,
+    dataSources,
 }, ref) {
     const [messages, setMessages] = useState<z.infer<typeof CopilotMessage>[]>([]);
     const [discardContext, setDiscardContext] = useState(false);
@@ -63,7 +66,8 @@ const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({
     } = useCopilot({
         projectId,
         workflow: workflowRef.current,
-        context: effectiveContext
+        context: effectiveContext,
+        dataSources: dataSources
     });
 
     // Store latest start/cancel functions in refs
@@ -129,7 +133,8 @@ const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({
     }, [messages, onCopyJson]);
 
     useImperativeHandle(ref, () => ({
-        handleCopyChat
+        handleCopyChat,
+        handleUserMessage
     }), [handleCopyChat]);
 
     return (
@@ -190,23 +195,27 @@ const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({
     );
 });
 
-export function Copilot({
-    projectId,
-    workflow,
-    chatContext = undefined,
-    dispatch,
-    isInitialState = false,
-}: {
+App.displayName = 'App';
+
+export const Copilot = forwardRef<{ handleUserMessage: (message: string) => void }, {
     projectId: string;
     workflow: z.infer<typeof Workflow>;
     chatContext?: z.infer<typeof CopilotChatContext>;
     dispatch: (action: WorkflowDispatch) => void;
     isInitialState?: boolean;
-}) {
+    dataSources?: z.infer<typeof DataSource>[];
+}>(({
+    projectId,
+    workflow,
+    chatContext = undefined,
+    dispatch,
+    isInitialState = false,
+    dataSources,
+}, ref) => {
     const [copilotKey, setCopilotKey] = useState(0);
     const [showCopySuccess, setShowCopySuccess] = useState(false);
     const [messages, setMessages] = useState<z.infer<typeof CopilotMessage>[]>([]);
-    const appRef = useRef<{ handleCopyChat: () => void }>(null);
+    const appRef = useRef<{ handleCopyChat: () => void; handleUserMessage: (message: string) => void }>(null);
 
     function handleNewChat() {
         setCopilotKey(prev => prev + 1);
@@ -222,6 +231,16 @@ export function Copilot({
         }, 2000);
     }
 
+    // Expose handleUserMessage through ref
+    useImperativeHandle(ref, () => ({
+        handleUserMessage: (message: string) => {
+            const app = appRef.current as any;
+            if (app?.handleUserMessage) {
+                app.handleUserMessage(message);
+            }
+        }
+    }), []);
+
     return (
         <Panel variant="copilot"
             tourTarget="copilot"
@@ -277,9 +296,12 @@ export function Copilot({
                     onCopyJson={handleCopyJson}
                     onMessagesChange={setMessages}
                     isInitialState={isInitialState}
+                    dataSources={dataSources}
                 />
             </div>
         </Panel>
     );
-}
+});
+
+Copilot.displayName = 'Copilot';
 

apps/rowboat/app/projects/[projectId]/copilot/use-copilot.tsx

@@ -2,12 +2,14 @@ import { useCallback, useRef, useState } from "react";
 import { getCopilotResponseStream } from "@/app/actions/copilot_actions";
 import { CopilotMessage } from "@/app/lib/types/copilot_types";
 import { Workflow } from "@/app/lib/types/workflow_types";
+import { DataSource } from "@/app/lib/types/datasource_types";
 import { z } from "zod";
 
 interface UseCopilotParams {
     projectId: string;
     workflow: z.infer<typeof Workflow>;
     context: any;
+    dataSources?: z.infer<typeof DataSource>[];
 }
 
 interface UseCopilotResult {
@@ -21,7 +23,7 @@ interface UseCopilotResult {
     cancel: () => void;
 }
 
-export function useCopilot({ projectId, workflow, context }: UseCopilotParams): UseCopilotResult {
+export function useCopilot({ projectId, workflow, context, dataSources }: UseCopilotParams): UseCopilotResult {
     const [streamingResponse, setStreamingResponse] = useState('');
     const [loading, setLoading] = useState(false);
     const [error, setError] = useState<string | null>(null);
@@ -41,7 +43,7 @@ export function useCopilot({ projectId, workflow, context }: UseCopilotParams):
         setLoading(true);
 
         try {
-            const res = await getCopilotResponseStream(projectId, messages, workflow, context || null);
+            const res = await getCopilotResponseStream(projectId, messages, workflow, context || null, dataSources);
             const eventSource = new EventSource(`/api/copilot-stream-response/${res.streamId}`);
 
             eventSource.onmessage = (event) => {
@@ -71,7 +73,7 @@ export function useCopilot({ projectId, workflow, context }: UseCopilotParams):
             setError('Failed to initiate stream');
             setLoading(false);
         }
-    }, [projectId, workflow, context]);
+    }, [projectId, workflow, context, dataSources]);
 
     const cancel = useCallback(() => {
         cancelRef.current?.();

apps/rowboat/app/projects/[projectId]/entities/agent_config.tsx

@@ -22,6 +22,7 @@ import { EditableField } from "@/app/lib/components/editable-field";
 import { USE_TRANSFER_CONTROL_OPTIONS } from "@/app/lib/feature_flags";
 import { Input } from "@/components/ui/input";
 import { Info } from "lucide-react";
+import { useCopilot } from "../copilot/use-copilot";
 
 // Common section header styles
 const sectionHeaderStyles = "text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400";
@@ -30,7 +31,7 @@ const sectionHeaderStyles = "text-xs font-medium uppercase tracking-wider text-g
 const textareaStyles = "rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500";
 
 // Add this type definition after the imports
-type TabType = 'instructions' | 'examples' | 'configurations';
+type TabType = 'instructions' | 'examples' | 'configurations' | 'rag';
 
 export function AgentConfig({
     projectId,
@@ -44,6 +45,7 @@ export function AgentConfig({
     handleUpdate,
     handleClose,
     useRag,
+    triggerCopilotChat,
 }: {
     projectId: string,
     workflow: z.infer<typeof Workflow>,
@@ -56,6 +58,7 @@ export function AgentConfig({
     handleUpdate: (agent: z.infer<typeof WorkflowAgent>) => void,
     handleClose: () => void,
     useRag: boolean,
+    triggerCopilotChat: (message: string) => void,
 }) {
     const [isAdvancedConfigOpen, setIsAdvancedConfigOpen] = useState(false);
     const [showGenerateModal, setShowGenerateModal] = useState(false);
@@ -65,11 +68,42 @@ export function AgentConfig({
     const [localName, setLocalName] = useState(agent.name);
     const [nameError, setNameError] = useState<string | null>(null);
     const [activeTab, setActiveTab] = useState<TabType>('instructions');
+    const [showRagCta, setShowRagCta] = useState(false);
+    const [previousRagSources, setPreviousRagSources] = useState<string[]>([]);
     
+    const {
+        start: startCopilotChat,
+    } = useCopilot({
+        projectId,
+        workflow,
+        context: null,
+        dataSources
+    });
+
     useEffect(() => {
         setLocalName(agent.name);
     }, [agent.name]);
 
+    // Track changes in RAG datasources
+    useEffect(() => {
+        const currentSources = agent.ragDataSources || [];
+        // Show CTA when transitioning from 0 to 1 datasource
+        if (currentSources.length === 1 && previousRagSources.length === 0) {
+            setShowRagCta(true);
+        }
+        // Hide CTA when all datasources are deleted
+        if (currentSources.length === 0) {
+            setShowRagCta(false);
+        }
+        setPreviousRagSources(currentSources);
+    }, [agent.ragDataSources, previousRagSources.length]);
+
+    const handleUpdateInstructions = async () => {
+        const message = `Update the instructions for agent "${agent.name}" to use the rag tool (rag_search) since data sources have been added. If this has already been done, do not take any action, but let me know.`;
+        triggerCopilotChat(message);
+        setShowRagCta(false);
+    };
+
     // Add effect to handle control type update when transfer control is disabled
     useEffect(() => {
         if (!USE_TRANSFER_CONTROL_OPTIONS && agent.controlType !== 'retain') {
@@ -152,7 +186,7 @@ export function AgentConfig({
             <div className="flex flex-col gap-6 p-4 h-[calc(100vh-100px)] min-h-0 flex-1">
                 {/* Tabs */}
                 <div className="flex border-b border-gray-200 dark:border-gray-700">
-                    {(['instructions', 'examples', 'configurations'] as TabType[]).map((tab) => (
+                    {(['instructions', 'examples', 'configurations', 'rag'] as TabType[]).map((tab) => (
                         <button
                             key={tab}
                             onClick={() => setActiveTab(tab)}
@@ -163,7 +197,7 @@ export function AgentConfig({
                                     : "text-gray-500 hover:text-gray-700 dark:text-gray-400 dark:hover:text-gray-300"
                             )}
                         >
-                            {tab.charAt(0).toUpperCase() + tab.slice(1)}
+                            {tab === 'rag' ? 'RAG' : tab.charAt(0).toUpperCase() + tab.slice(1)}
                         </button>
                     ))}
                 </div>
@@ -449,169 +483,6 @@ export function AgentConfig({
                                 />
                             </div>
 
-                            {useRag && (
-                                <div className="space-y-4">
-                                    <label className={sectionHeaderStyles}>
-                                        RAG
-                                    </label>
-                                    <div className="flex flex-col gap-3">
-                                        <div>
-                                            <Select
-                                                variant="bordered"
-                                                placeholder="Add data source"
-                                                size="sm"
-                                                className="w-64"
-                                                onSelectionChange={(keys) => {
-                                                    const key = keys.currentKey as string;
-                                                    if (key) {
-                                                        handleUpdate({
-                                                            ...agent,
-                                                            ragDataSources: [...(agent.ragDataSources || []), key]
-                                                        });
-                                                    }
-                                                }}
-                                                startContent={<PlusIcon className="w-4 h-4 text-gray-500" />}
-                                            >
-                                                {dataSources
-                                                    .filter((ds) => !(agent.ragDataSources || []).includes(ds._id))
-                                                    .map((ds) => (
-                                                        <SelectItem key={ds._id}>
-                                                            {ds.name}
-                                                        </SelectItem>
-                                                    ))
-                                                }
-                                            </Select>
-                                        </div>
-
-                                        <div className="flex flex-col gap-2">
-                                            {(agent.ragDataSources || []).map((source) => {
-                                                const ds = dataSources.find((ds) => ds._id === source);
-                                                return (
-                                                    <div 
-                                                        key={source}
-                                                        className="flex items-center justify-between p-3 rounded-lg border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 transition-colors"
-                                                    >
-                                                        <div className="flex items-center gap-3">
-                                                            <div className="flex items-center justify-center w-8 h-8 rounded-md bg-indigo-50 dark:bg-indigo-900/20">
-                                                                <svg 
-                                                                    className="w-4 h-4 text-indigo-600 dark:text-indigo-400" 
-                                                                    fill="none" 
-                                                                    viewBox="0 0 24 24" 
-                                                                    stroke="currentColor"
-                                                                >
-                                                                    <path 
-                                                                        strokeLinecap="round" 
-                                                                        strokeLinejoin="round" 
-                                                                        strokeWidth={2} 
-                                                                        d="M19 11H5m14 0a2 2 0 012 2v6a2 2 0 01-2 2H5a2 2 0 01-2-2v-6a2 2 0 012-2m14 0V9a2 2 0 00-2-2M5 11V9a2 2 0 012-2m0 0V5a2 2 0 012-2h6a2 2 0 012 2v2M7 7h10" 
-                                                                    />
-                                                                </svg>
-                                                            </div>
-                                                            <div className="flex flex-col">
-                                                                <span className="text-sm font-medium text-gray-900 dark:text-gray-100">
-                                                                    {ds?.name || "Unknown"}
-                                                                </span>
-                                                                <span className="text-xs text-gray-500 dark:text-gray-400">
-                                                                    Data Source
-                                                                </span>
-                                                            </div>
-                                                        </div>
-                                                        <CustomButton
-                                                            variant="tertiary"
-                                                            size="sm"
-                                                            className="text-gray-500 hover:text-red-500 hover:bg-red-50 dark:hover:bg-red-900/20"
-                                                            onClick={() => {
-                                                                const newSources = agent.ragDataSources?.filter((s) => s !== source);
-                                                                handleUpdate({
-                                                                    ...agent,
-                                                                    ragDataSources: newSources
-                                                                });
-                                                            }}
-                                                            startContent={<Trash2 className="w-4 h-4" />}
-                                                        >
-                                                            Remove
-                                                        </CustomButton>
-                                                    </div>
-                                                );
-                                            })}
-                                        </div>
-
-                                        {agent.ragDataSources !== undefined && agent.ragDataSources.length > 0 && (
-                                            <>
-                                                <div className="mt-4">
-                                                    <button
-                                                        onClick={() => setIsAdvancedConfigOpen(!isAdvancedConfigOpen)}
-                                                        className="flex items-center gap-2 text-xs font-medium text-gray-500 dark:text-gray-400 uppercase hover:text-gray-700 dark:hover:text-gray-300 transition-colors"
-                                                    >
-                                                        {isAdvancedConfigOpen ? 
-                                                            <ChevronDown className="w-4 h-4 text-gray-400" /> : 
-                                                            <ChevronRight className="w-4 h-4 text-gray-400" />
-                                                        }
-                                                        Advanced RAG configuration
-                                                    </button>
-                                                    
-                                                    {isAdvancedConfigOpen && (
-                                                        <div className="mt-3 ml-4 p-4 rounded-lg border border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-800/50">
-                                                            <div className="grid gap-6">
-                                                                <div className="space-y-2">
-                                                                    <label className={sectionHeaderStyles}>
-                                                                        Return type
-                                                                    </label>
-                                                                    <div className="flex gap-4">
-                                                                        {["chunks", "content"].map((type) => (
-                                                                            <button
-                                                                                key={type}
-                                                                                onClick={() => handleUpdate({
-                                                                                    ...agent,
-                                                                                    ragReturnType: type as z.infer<typeof WorkflowAgent>['ragReturnType']
-                                                                                })}
-                                                                                className={clsx(
-                                                                                    "px-4 py-2 rounded-lg text-sm font-medium transition-colors",
-                                                                                    agent.ragReturnType === type
-                                                                                        ? "bg-indigo-50 dark:bg-indigo-900/20 text-indigo-600 dark:text-indigo-400 border-2 border-indigo-200 dark:border-indigo-800"
-                                                                                        : "bg-white dark:bg-gray-800 text-gray-600 dark:text-gray-400 border border-gray-200 dark:border-gray-700 hover:border-gray-300 dark:hover:border-gray-600"
-                                                                                )}
-                                                                            >
-                                                                                {type.charAt(0).toUpperCase() + type.slice(1)}
-                                                                            </button>
-                                                                        ))}
-                                                                    </div>
-                                                                </div>
-
-                                                                <div className="space-y-2">
-                                                                    <label className={sectionHeaderStyles}>
-                                                                        Number of matches
-                                                                    </label>
-                                                                    <div className="flex items-center gap-3">
-                                                                        <input
-                                                                            type="number"
-                                                                            min="1"
-                                                                            max="20"
-                                                                            className="w-24 px-3 py-2 rounded-lg border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 text-sm focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 focus:border-indigo-500 dark:focus:border-indigo-400"
-                                                                            value={agent.ragK}
-                                                                            onChange={(e) => handleUpdate({
-                                                                                ...agent,
-                                                                                ragK: parseInt(e.target.value)
-                                                                            })}
-                                                                        />
-                                                                        <span className="text-sm text-gray-500 dark:text-gray-400">
-                                                                            matches
-                                                                        </span>
-                                                                    </div>
-                                                                    <p className="text-xs text-gray-500 dark:text-gray-400">
-                                                                        Number of relevant chunks to retrieve (1-20)
-                                                                    </p>
-                                                                </div>
-                                                            </div>
-                                                        </div>
-                                                    )}
-                                                </div>
-                                            </>
-                                        )}
-                                    </div>
-                                </div>
-                            )}
-
                             <div className="space-y-4">
                                 <div className="flex items-center">
                                     <label className={sectionHeaderStyles}>
@@ -697,6 +568,182 @@ export function AgentConfig({
                             )}
                         </div>
                     )}
+
+                    {activeTab === 'rag' && useRag && (
+                        <div className="space-y-6">
+                            <div className="flex flex-col gap-3">
+                                <div className="space-y-2">
+                                    <label className={sectionHeaderStyles}>
+                                        DATA SOURCES
+                                    </label>
+                                    <div className="flex items-center gap-3">
+                                        <Select
+                                            variant="bordered"
+                                            placeholder="Add data source"
+                                            size="sm"
+                                            className="w-64"
+                                            onSelectionChange={(keys) => {
+                                                const key = keys.currentKey as string;
+                                                if (key) {
+                                                    handleUpdate({
+                                                        ...agent,
+                                                        ragDataSources: [...(agent.ragDataSources || []), key]
+                                                    });
+                                                }
+                                            }}
+                                            startContent={<PlusIcon className="w-4 h-4 text-gray-500" />}
+                                        >
+                                            {dataSources
+                                                .filter((ds) => !(agent.ragDataSources || []).includes(ds._id))
+                                                .map((ds) => (
+                                                    <SelectItem key={ds._id}>
+                                                        {ds.name}
+                                                    </SelectItem>
+                                                ))
+                                            }
+                                        </Select>
+
+                                        {showRagCta && (
+                                            <CustomButton
+                                                variant="primary"
+                                                size="sm"
+                                                onClick={handleUpdateInstructions}
+                                                className="whitespace-nowrap"
+                                            >
+                                                Update Instructions
+                                            </CustomButton>
+                                        )}
+                                    </div>
+                                </div>
+
+                                <div className="flex flex-col gap-2">
+                                    {(agent.ragDataSources || []).map((source) => {
+                                        const ds = dataSources.find((ds) => ds._id === source);
+                                        return (
+                                            <div 
+                                                key={source}
+                                                className="flex items-center justify-between p-3 rounded-lg border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 transition-colors"
+                                            >
+                                                <div className="flex items-center gap-3">
+                                                    <div className="flex items-center justify-center w-8 h-8 rounded-md bg-indigo-50 dark:bg-indigo-900/20">
+                                                        <svg 
+                                                            className="w-4 h-4 text-indigo-600 dark:text-indigo-400" 
+                                                            fill="none" 
+                                                            viewBox="0 0 24 24" 
+                                                            stroke="currentColor"
+                                                        >
+                                                            <path 
+                                                                strokeLinecap="round" 
+                                                                strokeLinejoin="round" 
+                                                                strokeWidth={2} 
+                                                                d="M19 11H5m14 0a2 2 0 012 2v6a2 2 0 01-2 2H5a2 2 0 01-2-2v-6a2 2 0 012-2m14 0V9a2 2 0 00-2-2M5 11V9a2 2 0 012-2m0 0V5a2 2 0 012-2h6a2 2 0 012 2v2M7 7h10" 
+                                                            />
+                                                        </svg>
+                                                    </div>
+                                                    <div className="flex flex-col">
+                                                        <span className="text-sm font-medium text-gray-900 dark:text-gray-100">
+                                                            {ds?.name || "Unknown"}
+                                                        </span>
+                                                        <span className="text-xs text-gray-500 dark:text-gray-400">
+                                                            Data Source
+                                                        </span>
+                                                    </div>
+                                                </div>
+                                                <CustomButton
+                                                    variant="tertiary"
+                                                    size="sm"
+                                                    className="text-gray-500 hover:text-red-500 hover:bg-red-50 dark:hover:bg-red-900/20"
+                                                    onClick={() => {
+                                                        const newSources = agent.ragDataSources?.filter((s) => s !== source);
+                                                        handleUpdate({
+                                                            ...agent,
+                                                            ragDataSources: newSources
+                                                        });
+                                                    }}
+                                                    startContent={<Trash2 className="w-4 h-4" />}
+                                                >
+                                                    Remove
+                                                </CustomButton>
+                                            </div>
+                                        );
+                                    })}
+                                </div>
+
+                                {agent.ragDataSources !== undefined && agent.ragDataSources.length > 0 && (
+                                    <>
+                                        <div className="mt-4">
+                                            <button
+                                                onClick={() => setIsAdvancedConfigOpen(!isAdvancedConfigOpen)}
+                                                className="flex items-center gap-2 text-xs font-medium text-gray-500 dark:text-gray-400 uppercase hover:text-gray-700 dark:hover:text-gray-300 transition-colors"
+                                            >
+                                                {isAdvancedConfigOpen ? 
+                                                    <ChevronDown className="w-4 h-4 text-gray-400" /> : 
+                                                    <ChevronRight className="w-4 h-4 text-gray-400" />
+                                                }
+                                                Advanced RAG configuration
+                                            </button>
+                                            
+                                            {isAdvancedConfigOpen && (
+                                                <div className="mt-3 ml-4 p-4 rounded-lg border border-gray-200 dark:border-gray-700 bg-gray-50 dark:bg-gray-800/50">
+                                                    <div className="grid gap-6">
+                                                        <div className="space-y-2">
+                                                            <label className={sectionHeaderStyles}>
+                                                                Return type
+                                                            </label>
+                                                            <div className="flex gap-4">
+                                                                {["chunks", "content"].map((type) => (
+                                                                    <button
+                                                                        key={type}
+                                                                        onClick={() => handleUpdate({
+                                                                            ...agent,
+                                                                            ragReturnType: type as z.infer<typeof WorkflowAgent>['ragReturnType']
+                                                                        })}
+                                                                        className={clsx(
+                                                                            "px-4 py-2 rounded-lg text-sm font-medium transition-colors",
+                                                                            agent.ragReturnType === type
+                                                                                ? "bg-indigo-50 dark:bg-indigo-900/20 text-indigo-600 dark:text-indigo-400 border-2 border-indigo-200 dark:border-indigo-800"
+                                                                                : "bg-white dark:bg-gray-800 text-gray-600 dark:text-gray-400 border border-gray-200 dark:border-gray-700 hover:border-gray-300 dark:hover:border-gray-600"
+                                                                        )}
+                                                                    >
+                                                                        {type.charAt(0).toUpperCase() + type.slice(1)}
+                                                                    </button>
+                                                                ))}
+                                                            </div>
+                                                        </div>
+
+                                                        <div className="space-y-2">
+                                                            <label className={sectionHeaderStyles}>
+                                                                Number of matches
+                                                            </label>
+                                                            <div className="flex items-center gap-3">
+                                                                <input
+                                                                    type="number"
+                                                                    min="1"
+                                                                    max="20"
+                                                                    className="w-24 px-3 py-2 rounded-lg border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 text-sm focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 focus:border-indigo-500 dark:focus:border-indigo-400"
+                                                                    value={agent.ragK}
+                                                                    onChange={(e) => handleUpdate({
+                                                                        ...agent,
+                                                                        ragK: parseInt(e.target.value)
+                                                                    })}
+                                                                />
+                                                                <span className="text-sm text-gray-500 dark:text-gray-400">
+                                                                    matches
+                                                                </span>
+                                                            </div>
+                                                            <p className="text-xs text-gray-500 dark:text-gray-400">
+                                                                Number of relevant chunks to retrieve (1-20)
+                                                            </p>
+                                                        </div>
+                                                    </div>
+                                                </div>
+                                            )}
+                                        </div>
+                                    </>
+                                )}
+                            </div>
+                        </div>
+                    )}
                 </div>
 
                 <PreviewModalProvider>

apps/rowboat/app/projects/[projectId]/playground/components/messages.tsx

@@ -6,7 +6,7 @@ import { Workflow } from "@/app/lib/types/workflow_types";
 import { WorkflowTool } from "@/app/lib/types/workflow_types";
 import MarkdownContent from "@/app/lib/components/markdown-content";
 import { apiV1 } from "rowboat-shared";
-import { MessageSquareIcon, EllipsisIcon, CircleCheckIcon, ChevronRightIcon, ChevronDownIcon, ChevronUpIcon, XIcon, PlusIcon } from "lucide-react";
+import { MessageSquareIcon, EllipsisIcon, CircleCheckIcon, ChevronRightIcon, ChevronDownIcon, ChevronUpIcon, XIcon, PlusIcon, CodeIcon, CheckCircleIcon, FileTextIcon } from "lucide-react";
 import { TestProfile } from "@/app/lib/types/testing_types";
 import { ProfileContextBox } from "./profile-context-box";
 
@@ -245,7 +245,7 @@ function ClientToolCall({
     delta: number;
 }) {
     return (
-        <div className="self-start flex flex-col gap-1">
+        <div className="self-start flex flex-col gap-1 mb-4">
             {sender && (
                 <div className="text-gray-500 dark:text-gray-400 text-xs pl-1">
                     {sender}
@@ -258,18 +258,19 @@ function ClientToolCall({
                     <div className="flex flex-col gap-1">
                         <div className="shrink-0 flex gap-2 items-center">
                             {!availableResult && <Spinner size="sm" />}
-                            {availableResult && <CircleCheckIcon size={16} />}
-                            <div className="font-semibold text-sm">
-                                Function Call: <code className="bg-gray-100 dark:bg-neutral-800 px-2 py-0.5 rounded font-mono">
+                            {availableResult && <CheckCircleIcon size={16} className="text-green-500" />}
+                            <div className="flex items-center font-semibold text-sm gap-2">
+                                <span>Function Call:</span>
+                                <span className="px-2 py-0.5 rounded-full bg-purple-50 text-purple-800 dark:bg-purple-900/30 dark:text-purple-100 font-bold text-sm align-middle">
                                     {toolCall.function.name}
-                                </code>
+                                </span>
                             </div>
                         </div>
                     </div>
 
                     <div className="flex flex-col gap-2">
-                        <ExpandableContent label="Params" content={toolCall.function.arguments} expanded={false} />
-                        {availableResult && <ExpandableContent label="Result" content={availableResult.content} expanded={false} />}
+                        <ExpandableContent label="Params" content={toolCall.function.arguments} expanded={false} icon={<CodeIcon size={14} />} />
+                        {availableResult && <ExpandableContent label="Result" content={availableResult.content} expanded={false} icon={<FileTextIcon size={14} className="text-blue-500" />} />}
                     </div>
                 </div>
             </div>
@@ -280,11 +281,13 @@ function ClientToolCall({
 function ExpandableContent({
     label,
     content,
-    expanded = false
+    expanded = false,
+    icon
 }: {
     label: string,
     content: string | object | undefined,
-    expanded?: boolean
+    expanded?: boolean,
+    icon?: React.ReactNode
 }) {
     const [isExpanded, setIsExpanded] = useState(expanded);
 
@@ -314,6 +317,7 @@ function ExpandableContent({
         <div className='flex gap-1 items-start cursor-pointer text-gray-500 dark:text-gray-400' onClick={toggleExpanded}>
             {!isExpanded && <ChevronRightIcon size={16} />}
             {isExpanded && <ChevronDownIcon size={16} />}
+            {icon && <span className="mr-1">{icon}</span>}
             <div className='text-left break-all text-xs'>{label}</div>
         </div>
         {isExpanded && (
@@ -322,7 +326,10 @@ function ExpandableContent({
                     <MarkdownContent content={content as string} />
                 </div>
             ) : (
-                <pre className='text-sm font-mono bg-gray-100 dark:bg-gray-800 p-2 rounded break-all whitespace-pre-wrap overflow-x-auto text-gray-900 dark:text-gray-100'>
+                <pre
+                  className="text-xs leading-snug bg-zinc-50 dark:bg-zinc-800 text-zinc-700 dark:text-zinc-200 rounded-lg px-2 py-1 overflow-x-auto font-mono shadow-sm border border-zinc-100 dark:border-zinc-700"
+                  style={{ fontFamily: "'JetBrains Mono', 'Fira Mono', 'Menlo', 'Consolas', 'Liberation Mono', monospace" }}
+                >
                     {formattedContent}
                 </pre>
             )

apps/rowboat/app/projects/[projectId]/sources/[sourceId]/source-page.tsx

@@ -10,10 +10,14 @@ import { DataSourceIcon } from "../../../../lib/components/datasource-icon";
 import { z } from "zod";
 import { ScrapeSource } from "../components/scrape-source";
 import { FilesSource } from "../components/files-source";
-import { getDataSource } from "../../../../actions/datasource_actions";
+import { getDataSource, updateDataSource } from "../../../../actions/datasource_actions";
 import { TextSource } from "../components/text-source";
 import { Panel } from "@/components/common/panel-common";
 import { Section, SectionRow, SectionLabel, SectionContent } from "../components/section";
+import Link from "next/link";
+import { BackIcon } from "../../../../lib/components/icons";
+import { Textarea } from "@/components/ui/textarea";
+import { CheckIcon } from "lucide-react";
 
 export function SourcePage({
     sourceId,
@@ -24,6 +28,7 @@ export function SourcePage({
 }) {
     const [source, setSource] = useState<WithStringId<z.infer<typeof DataSource>> | null>(null);
     const [isLoading, setIsLoading] = useState(true);
+    const [showSaveSuccess, setShowSaveSuccess] = useState(false);
 
     async function handleReload() {
         setIsLoading(true);
@@ -95,6 +100,15 @@ export function SourcePage({
         <Panel title={source.name.toUpperCase()}>
             <div className="h-full overflow-auto px-4 py-4">
                 <div className="max-w-[768px] mx-auto space-y-6">
+                    <div className="flex items-center gap-2 mb-4">
+                        <Link 
+                            href={`/projects/${projectId}/sources`}
+                            className="flex items-center gap-2 text-sm text-gray-600 hover:text-gray-900 dark:text-gray-400 dark:hover:text-gray-100"
+                        >
+                            <BackIcon size={16} />
+                            <span>Back to sources</span>
+                        </Link>
+                    </div>
                     <Section
                         title="Details"
                         description="Basic information about this data source."
@@ -111,6 +125,57 @@ export function SourcePage({
                                 </SectionContent>
                             </SectionRow>
                             
+                            <SectionRow>
+                                <SectionLabel>Name</SectionLabel>
+                                <SectionContent>
+                                    <div className="text-sm text-gray-900 dark:text-gray-100">
+                                        {source.name}
+                                    </div>
+                                </SectionContent>
+                            </SectionRow>
+
+                            <SectionRow>
+                                <SectionLabel className="pt-3">Description</SectionLabel>
+                                <SectionContent>
+                                    <form
+                                        action={async (formData: FormData) => {
+                                            const description = formData.get('description') as string;
+                                            await updateDataSource({
+                                                projectId,
+                                                sourceId,
+                                                description,
+                                            });
+                                            handleReload();
+                                            setShowSaveSuccess(true);
+                                            setTimeout(() => setShowSaveSuccess(false), 2000);
+                                        }}
+                                        className="w-full"
+                                    >
+                                        <Textarea
+                                            name="description"
+                                            defaultValue={source.description || ''}
+                                            placeholder="Add a description for this data source"
+                                            rows={2}
+                                            className="w-full rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
+                                        />
+                                        <div className="flex items-center gap-2 mt-2">
+                                            <button
+                                                type="submit"
+                                                className="text-sm text-blue-600 hover:text-blue-700 dark:text-blue-400 dark:hover:text-blue-300"
+                                            >
+                                                Save
+                                            </button>
+                                            {showSaveSuccess && (
+                                                <div className="flex items-center gap-1 text-sm text-green-600 dark:text-green-400">
+                                                    <CheckIcon className="w-4 h-4" />
+                                                    <span>Saved</span>
+                                                </div>
+                                            )}
+                                        </div>
+                                    </form>
+                                </SectionContent>
+                            </SectionRow>
+
                             <SectionRow>
                                 <SectionLabel>Type</SectionLabel>
                                 <SectionContent>
@@ -119,9 +184,13 @@ export function SourcePage({
                                             <DataSourceIcon type="urls" />
                                             <div>Specify URLs</div>
                                         </>}
-                                        {source.data.type === 'files' && <>
+                                        {source.data.type === 'files_local' && <>
                                             <DataSourceIcon type="files" />
-                                            <div>File upload</div>
+                                            <div>File upload (local)</div>
+                                        </>}
+                                        {source.data.type === 'files_s3' && <>
+                                            <DataSourceIcon type="files" />
+                                            <div>File upload (S3)</div>
                                         </>}
                                         {source.data.type === 'text' && <>
                                             <DataSourceIcon type="text" />
@@ -131,12 +200,15 @@ export function SourcePage({
                                 </SectionContent>
                             </SectionRow>
 
-                            <SectionRow>
-                                <SectionLabel>Source</SectionLabel>
-                                <SectionContent>
-                                    <SourceStatus status={source.status} projectId={projectId} />
-                                </SectionContent>
-                            </SectionRow>
+                            {/* Only show status when it exists */}
+                            {source.status && (
+                                <SectionRow>
+                                    <SectionLabel>Status</SectionLabel>
+                                    <SectionContent>
+                                        <SourceStatus status={source.status} projectId={projectId} />
+                                    </SectionContent>
+                                </SectionRow>
+                            )}
                         </div>
                     </Section>
 
@@ -148,11 +220,12 @@ export function SourcePage({
                             handleReload={handleReload} 
                         />
                     }
-                    {source.data.type === 'files' && 
+                    {(source.data.type === 'files_local' || source.data.type === 'files_s3') && 
                         <FilesSource 
                             projectId={projectId} 
                             dataSource={source} 
                             handleReload={handleReload} 
+                            type={source.data.type}
                         />
                     }
                     {source.data.type === 'text' && 

apps/rowboat/app/projects/[projectId]/sources/components/files-source.tsx

@@ -46,7 +46,7 @@ function FileListItem({
         }
     };
 
-    if (file.data.type !== 'file') {
+    if (file.data.type !== 'file_local' && file.data.type !== 'file_s3') {
         return null;
     }
 
@@ -180,10 +180,12 @@ export function FilesSource({
     projectId,
     dataSource,
     handleReload,
+    type,
 }: {
     projectId: string,
     dataSource: WithStringId<z.infer<typeof DataSource>>,
     handleReload: () => void;
+    type: 'files_local' | 'files_s3';
 }) {
     const [uploading, setUploading] = useState(false);
     const [fileListKey, setFileListKey] = useState(0);
@@ -199,7 +201,7 @@ export function FilesSource({
 
             // Upload files in parallel
             await Promise.all(acceptedFiles.map(async (file, index) => {
-                await fetch(urls[index].presignedUrl, {
+                await fetch(urls[index].uploadUrl, {
                     method: 'PUT',
                     body: file,
                     headers: {
@@ -209,20 +211,40 @@ export function FilesSource({
             }));
 
             // After successful uploads, update the database with file information
-            await addDocsToDataSource({
-                projectId,
-                sourceId: dataSource._id,
-                docData: acceptedFiles.map((file, index) => ({
+            let docData: {
+                _id: string,
+                name: string,
+                data: z.infer<typeof DataSourceDoc>['data']
+            }[] = [];
+            if (type === 'files_s3') {
+                docData = acceptedFiles.map((file, index) => ({
                     _id: urls[index].fileId,
                     name: file.name,
                     data: {
-                        type: 'file',
+                        type: 'file_s3' as const,
                         name: file.name,
                         size: file.size,
                         mimeType: file.type,
-                        s3Key: urls[index].s3Key,
+                        s3Key: urls[index].path,
                     },
-                })),
+                }));
+            } else {
+                docData = acceptedFiles.map((file, index) => ({
+                    _id: urls[index].fileId,
+                    name: file.name,
+                    data: {
+                        type: 'file_local' as const,
+                        name: file.name,
+                        size: file.size,
+                        mimeType: file.type,
+                    },
+                }));
+            }
+
+            await addDocsToDataSource({
+                projectId,
+                sourceId: dataSource._id,
+                docData,
             });
 
             handleReload();
@@ -233,22 +255,22 @@ export function FilesSource({
         } finally {
             setUploading(false);
         }
-    }, [projectId, dataSource._id, handleReload]);
+    }, [projectId, dataSource._id, handleReload, type]);
 
     const { getRootProps, getInputProps, isDragActive } = useDropzone({
         onDrop,
         disabled: uploading,
         accept: {
             'application/pdf': ['.pdf'],
-            'text/plain': ['.txt'],
-            'application/msword': ['.doc'],
-            'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
+            // 'text/plain': ['.txt'],
+            // 'application/msword': ['.doc'],
+            // 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': ['.docx'],
         },
     });
 
     return (
-        <Section 
-            title="File Uploads" 
+        <Section
+            title="File Uploads"
             description="Upload and manage files for this data source."
         >
             <div className="space-y-8">
@@ -269,7 +291,7 @@ export function FilesSource({
                         <div className="space-y-2">
                             <p>Drag and drop files here, or click to select files</p>
                             <p className="text-sm text-gray-500 dark:text-gray-400">
-                                Supported file types: PDF, TXT, DOC, DOCX
+                                Only PDF files are supported for now.
                             </p>
                         </div>
                     )}

apps/rowboat/app/projects/[projectId]/sources/components/section.tsx

@@ -29,15 +29,15 @@ export function Section({ title, description, children, className }: SectionProp
 
 export function SectionRow({ children, className }: { children: ReactNode; className?: string }) {
     return (
-        <div className={`flex items-center gap-6 ${className || ''}`}>
+        <div className={`flex items-start gap-6 py-1 ${className || ''}`}>
             {children}
         </div>
     );
 }
 
-export function SectionLabel({ children }: { children: ReactNode }) {
+export function SectionLabel({ children, className }: { children: ReactNode; className?: string }) {
     return (
-        <div className="w-24 flex-shrink-0 text-sm text-gray-500 dark:text-gray-400">
+        <div className={`w-24 flex-shrink-0 text-sm text-gray-500 dark:text-gray-400 ${className || ''}`}>
             {children}
         </div>
     );

apps/rowboat/app/projects/[projectId]/sources/components/sources-list.tsx

@@ -71,84 +71,116 @@ export function SourcesList({ projectId }: { projectId: string }) {
                         <p className="mt-4 text-center">You have not added any data sources.</p>
                     )}
                     {!loading && sources.length > 0 && (
-                        <div className="border rounded-lg overflow-hidden">
-                            <table className="w-full">
-                                <thead className="bg-gray-50 dark:bg-gray-800/50">
-                                    <tr>
-                                        <th className="w-[30%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
-                                            Name
-                                        </th>
-                                        <th className="w-[20%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
-                                            Type
-                                        </th>
-                                        <th className="w-[35%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
-                                            Status
-                                        </th>
-                                        <th className="w-[15%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
-                                            Active
-                                        </th>
-                                    </tr>
-                                </thead>
-                                <tbody className="bg-white dark:bg-gray-800 divide-y divide-gray-200 dark:divide-gray-700">
-                                    {sources.map((source) => (
-                                        <tr 
-                                            key={source._id}
-                                            className="hover:bg-gray-50 dark:hover:bg-gray-750 transition-colors"
-                                        >
-                                            <td className="px-6 py-4 text-left">
-                                                <Link
-                                                    href={`/projects/${projectId}/sources/${source._id}`}
-                                                    size="lg"
-                                                    isBlock
-                                                    className="text-sm text-gray-900 dark:text-gray-100 hover:text-blue-600 dark:hover:text-blue-400 truncate block"
-                                                >
-                                                    {source.name}
-                                                </Link>
-                                            </td>
-                                            <td className="px-6 py-4 text-left">
-                                                {source.data.type == 'urls' && (
-                                                    <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
-                                                        <DataSourceIcon type="urls" />
-                                                        <div>List URLs</div>
-                                                    </div>
-                                                )}
-                                                {source.data.type == 'text' && (
-                                                    <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
-                                                        <DataSourceIcon type="text" />
-                                                        <div>Text</div>
-                                                    </div>
-                                                )}
-                                                {source.data.type == 'files' && (
-                                                    <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
-                                                        <DataSourceIcon type="files" />
-                                                        <div>Files</div>
-                                                    </div>
-                                                )}
-                                            </td>
-                                            <td className="px-6 py-4 text-left">
-                                                <div className="text-sm">
-                                                    <SelfUpdatingSourceStatus 
-                                                        sourceId={source._id} 
-                                                        projectId={projectId} 
-                                                        initialStatus={source.status} 
-                                                        compact={true} 
-                                                    />
-                                                </div>
-                                            </td>
-                                            <td className="px-6 py-4 text-left">
-                                                <ToggleSource 
-                                                    projectId={projectId} 
-                                                    sourceId={source._id} 
-                                                    active={source.active} 
-                                                    compact={true} 
-                                                    className="bg-default-100" 
-                                                />
-                                            </td>
+                        <>
+                            <div className="mb-6 p-4 bg-blue-50 dark:bg-blue-900/10 rounded-lg border border-blue-200 dark:border-blue-800">
+                                <div className="flex items-start gap-3">
+                                    <svg 
+                                        className="w-5 h-5 text-blue-500 mt-0.5" 
+                                        fill="none" 
+                                        stroke="currentColor" 
+                                        viewBox="0 0 24 24"
+                                    >
+                                        <path 
+                                            strokeLinecap="round" 
+                                            strokeLinejoin="round" 
+                                            strokeWidth={2} 
+                                            d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" 
+                                        />
+                                    </svg>
+                                    <div className="text-sm text-blue-700 dark:text-blue-300">
+                                        After creating data sources, go to the RAG tab inside individual agent settings to connect them to agents.
+                                    </div>
+                                </div>
+                            </div>
+                            <div className="border rounded-lg overflow-hidden">
+                                <table className="w-full">
+                                    <thead className="bg-gray-50 dark:bg-gray-800/50">
+                                        <tr>
+                                            <th className="w-[30%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                                Name
+                                            </th>
+                                            <th className="w-[20%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                                Type
+                                            </th>
+                                            {sources.some(source => source.status) && (
+                                                <th className="w-[35%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                                    Status
+                                                </th>
+                                            )}
+                                            <th className="w-[15%] px-6 py-3 text-left text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                                Active
+                                            </th>
                                         </tr>
-                                    ))}
-                                </tbody>
-                            </table>
-                        </div>
+                                    </thead>
+                                    <tbody className="bg-white dark:bg-gray-800 divide-y divide-gray-200 dark:divide-gray-700">
+                                        {sources.map((source) => (
+                                            <tr 
+                                                key={source._id}
+                                                className="hover:bg-gray-50 dark:hover:bg-gray-750 transition-colors"
+                                            >
+                                                <td className="px-6 py-4 text-left">
+                                                    <Link
+                                                        href={`/projects/${projectId}/sources/${source._id}`}
+                                                        size="lg"
+                                                        isBlock
+                                                        className="text-sm text-gray-900 dark:text-gray-100 hover:text-blue-600 dark:hover:text-blue-400 truncate block"
+                                                    >
+                                                        {source.name}
+                                                    </Link>
+                                                </td>
+                                                <td className="px-6 py-4 text-left">
+                                                    {source.data.type == 'urls' && (
+                                                        <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
+                                                            <DataSourceIcon type="urls" />
+                                                            <div>List URLs</div>
+                                                        </div>
+                                                    )}
+                                                    {source.data.type == 'text' && (
+                                                        <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
+                                                            <DataSourceIcon type="text" />
+                                                            <div>Text</div>
+                                                        </div>
+                                                    )}
+                                                    {source.data.type == 'files_local' && (
+                                                        <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
+                                                            <DataSourceIcon type="files" />
+                                                            <div>Files (Local)</div>
+                                                        </div>
+                                                    )}
+                                                    {source.data.type == 'files_s3' && (
+                                                        <div className="flex gap-2 items-center text-sm text-gray-600 dark:text-gray-300">
+                                                            <DataSourceIcon type="files" />
+                                                            <div>Files (S3)</div>
+                                                        </div>
+                                                    )}
+                                                </td>
+                                                {sources.some(source => source.status) && (
+                                                    <td className="px-6 py-4 text-left">
+                                                        <div className="text-sm">
+                                                            <SelfUpdatingSourceStatus 
+                                                                sourceId={source._id} 
+                                                                projectId={projectId} 
+                                                                initialStatus={source.status} 
+                                                                compact={true} 
+                                                            />
+                                                        </div>
+                                                    </td>
+                                                )}
+                                                <td className="px-6 py-4 text-left">
+                                                    <ToggleSource 
+                                                        projectId={projectId} 
+                                                        sourceId={source._id} 
+                                                        active={source.active} 
+                                                        compact={true} 
+                                                        className="bg-default-100" 
+                                                    />
+                                                </td>
+                                            </tr>
+                                        ))}
+                                    </tbody>
+                                </table>
+                            </div>
+                        </>
                     )}
                 </div>
             </div>

apps/rowboat/app/projects/[projectId]/sources/new/form.tsx

@@ -13,37 +13,51 @@ import { Panel } from "@/components/common/panel-common";
 export function Form({
     projectId,
     useRagUploads,
+    useRagS3Uploads,
     useRagScraping,
 }: {
     projectId: string;
     useRagUploads: boolean;
+    useRagS3Uploads: boolean;
     useRagScraping: boolean;
 }) {
     const [sourceType, setSourceType] = useState("");
     const router = useRouter();
 
-    const dropdownOptions = [
+    let dropdownOptions = [
         {
             key: "text",
             label: "Text",
             startContent: <DataSourceIcon type="text" />
         },
-        {
+    ];
+    if (useRagUploads) {
+        dropdownOptions.push({
+            key: "files_local",
+            label: "Upload files (Local)",
+            startContent: <DataSourceIcon type="files" />
+        });
+    }
+    if (useRagS3Uploads) {
+        dropdownOptions.push({
+            key: "files_s3",
+            label: "Upload files (S3)",
+            startContent: <DataSourceIcon type="files" />
+        });
+    }
+    if (useRagScraping) {
+        dropdownOptions.push({
             key: "urls",
             label: "Scrape URLs",
             startContent: <DataSourceIcon type="urls" />
-        },
-        {
-            key: "files",
-            label: "Upload files",
-            startContent: <DataSourceIcon type="files" />
-        }
-    ];
+        });
+    }
 
     async function createUrlsDataSource(formData: FormData) {
         const source = await createDataSource({
             projectId,
             name: formData.get('name') as string,
+            description: formData.get('description') as string,
             data: {
                 type: 'urls',
             },
@@ -72,10 +86,10 @@ export function Form({
         const source = await createDataSource({
             projectId,
             name: formData.get('name') as string,
+            description: formData.get('description') as string,
             data: {
-                type: 'files',
+                type: formData.get('type') as 'files_local' | 'files_s3',
             },
-            status: 'ready',
         });
 
         router.push(`/projects/${projectId}/sources/${source._id}`);
@@ -85,6 +99,7 @@ export function Form({
         const source = await createDataSource({
             projectId,
             name: formData.get('name') as string,
+            description: formData.get('description') as string,
             data: {
                 type: 'text',
             },
@@ -119,15 +134,31 @@ export function Form({
         >
             <div className="h-full overflow-auto px-4 py-4">
                 <div className="max-w-[768px] mx-auto flex flex-col gap-4">
+                    <div className="p-4 bg-blue-50 dark:bg-blue-900/10 rounded-lg border border-blue-200 dark:border-blue-800">
+                        <div className="flex items-start gap-3">
+                            <svg 
+                                className="w-5 h-5 text-blue-500 mt-0.5" 
+                                fill="none" 
+                                stroke="currentColor" 
+                                viewBox="0 0 24 24"
+                            >
+                                <path 
+                                    strokeLinecap="round" 
+                                    strokeLinejoin="round" 
+                                    strokeWidth={2} 
+                                    d="M13 16h-1v-4h-1m1-4h.01M21 12a9 9 0 11-18 0 9 9 0 0118 0z" 
+                                />
+                            </svg>
+                            <div className="text-sm text-blue-700 dark:text-blue-300">
+                                After creating data sources, go to the RAG tab inside individual agent settings to connect them to agents.
+                            </div>
+                        </div>
+                    </div>
                     <Dropdown
                         label="Select type"
                         value={sourceType}
                         onChange={setSourceType}
                         options={dropdownOptions}
-                        disabledKeys={[
-                            ...(useRagUploads ? [] : ['files']),
-                            ...(useRagScraping ? [] : ['urls']),
-                        ]}
                     />
 
                     {sourceType === "urls" && <form
@@ -158,6 +189,17 @@ export function Form({
                                 className="rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
                             />
                         </div>
+                        <div className="space-y-2">
+                            <label className="text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                Description
+                            </label>
+                            <Textarea
+                                name="description"
+                                placeholder="e.g. A collection of help articles from our documentation"
+                                rows={2}
+                                className="rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
+                            />
+                        </div>
                         <div className="bg-gray-50 dark:bg-gray-800/50 rounded-lg p-4 border border-gray-200 dark:border-gray-700">
                             <div className="flex items-center gap-2 mb-2 text-gray-700 dark:text-gray-300">
                                 <svg 
@@ -196,10 +238,11 @@ export function Form({
                         />
                     </form>}
 
-                    {sourceType === "files" && <form
+                    {(sourceType === "files_local" || sourceType === "files_s3") && <form
                         action={createFilesDataSource}
                         className="flex flex-col gap-4"
                     >
+                        <input type="hidden" name="type" value={sourceType} />
                         <div className="space-y-2">
                             <label className="text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
                                 Name
@@ -212,6 +255,17 @@ export function Form({
                                 className="rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
                             />
                         </div>
+                        <div className="space-y-2">
+                            <label className="text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                Description
+                            </label>
+                            <Textarea
+                                name="description"
+                                placeholder="e.g. A collection of documentation files"
+                                rows={2}
+                                className="rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
+                            />
+                        </div>
                         <div className="bg-gray-50 dark:bg-gray-800/50 rounded-lg p-4 border border-gray-200 dark:border-gray-700">
                             <div className="flex items-center gap-2 mb-2 text-gray-700 dark:text-gray-300">
                                 <svg 
@@ -271,6 +325,17 @@ export function Form({
                                 className="rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
                             />
                         </div>
+                        <div className="space-y-2">
+                            <label className="text-xs font-medium uppercase tracking-wider text-gray-500 dark:text-gray-400">
+                                Description
+                            </label>
+                            <Textarea
+                                name="description"
+                                placeholder="e.g. A collection of documentation for our product"
+                                rows={2}
+                                className="rounded-lg p-3 border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-800 hover:bg-gray-50 dark:hover:bg-gray-750 focus:shadow-inner focus:ring-2 focus:ring-indigo-500/20 dark:focus:ring-indigo-400/20 placeholder:text-gray-400 dark:placeholder:text-gray-500"
+                            />
+                        </div>
                         <FormStatusButton
                             props={{
                                 type: "submit",

apps/rowboat/app/projects/[projectId]/sources/new/page.tsx

@@ -1,7 +1,7 @@
 import { Metadata } from "next";
 import { Form } from "./form";
 import { redirect } from "next/navigation";
-import { USE_RAG, USE_RAG_UPLOADS, USE_RAG_SCRAPING } from "../../../../lib/feature_flags";
+import { USE_RAG, USE_RAG_UPLOADS, USE_RAG_S3_UPLOADS, USE_RAG_SCRAPING } from "../../../../lib/feature_flags";
 
 export const metadata: Metadata = {
     title: "Add data source"
@@ -20,6 +20,7 @@ export default async function Page({
         <Form
             projectId={params.projectId}
             useRagUploads={USE_RAG_UPLOADS}
+            useRagS3Uploads={USE_RAG_S3_UPLOADS}
             useRagScraping={USE_RAG_SCRAPING}
         />
     );

apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx

@@ -611,6 +611,16 @@ export function WorkflowEditor({
     const [isMcpImportModalOpen, setIsMcpImportModalOpen] = useState(false);
     const [isInitialState, setIsInitialState] = useState(true);
     const [showTour, setShowTour] = useState(true);
+    const copilotRef = useRef<{ handleUserMessage: (message: string) => void }>(null);
+
+    // Function to trigger copilot chat
+    const triggerCopilotChat = useCallback((message: string) => {
+        setShowCopilot(true);
+        // Small delay to ensure copilot is mounted
+        setTimeout(() => {
+            copilotRef.current?.handleUserMessage(message);
+        }, 100);
+    }, []);
 
     console.log(`workflow editor chat key: ${state.present.chatKey}`);
 
@@ -992,6 +1002,7 @@ export function WorkflowEditor({
                     handleUpdate={handleUpdateAgent.bind(null, state.present.selection.name)}
                     handleClose={handleUnselectAgent}
                     useRag={useRag}
+                    triggerCopilotChat={triggerCopilotChat}
                 />}
                 {state.present.selection?.type === "tool" && <ToolConfig
                     key={state.present.selection.name}
@@ -1020,6 +1031,7 @@ export function WorkflowEditor({
                         onResize={(size) => setCopilotWidth(size)}
                     >
                         <Copilot
+                            ref={copilotRef}
                             projectId={state.present.workflow.projectId}
                             workflow={state.present.workflow}
                             dispatch={dispatch}
@@ -1033,6 +1045,7 @@ export function WorkflowEditor({
                                 } : undefined
                             }
                             isInitialState={isInitialState}
+                            dataSources={dataSources}
                         />
                     </ResizablePanel>
                 </>

apps/rowboat/app/projects/layout/components/sidebar.tsx

@@ -147,7 +147,7 @@ export default function Sidebar({ projectId, useRag, useAuth, collapsed = false,
                             }
                           `}
                           disabled={isDisabled}
-                          data-tour-target={item.href === 'config' ? 'settings' : undefined}
+                          data-tour-target={item.href === 'config' ? 'settings' : item.href === 'sources' ? 'entity-data-sources' : undefined}
                         >
                           <Icon 
                             size={collapsed ? COLLAPSED_ICON_SIZE : EXPANDED_ICON_SIZE} 

apps/rowboat/app/scripts/delete_qdrant.ts

@@ -2,8 +2,10 @@ import '../lib/loadenv';
 import { qdrantClient } from '../lib/qdrant';
 
 (async () => {
-    await qdrantClient.deleteCollection('embeddings');
-
-    const { collections } = await qdrantClient.getCollections();
-    console.log(collections);
+    try {
+        const result = await qdrantClient.deleteCollection('embeddings');
+        console.log(`Delete qdrant collection 'embeddings' completed with result: ${result}`);
+    } catch (error) {
+        console.error(`Unable to delete qdrant collection 'embeddings': ${error}`);
+    }
 })();

apps/rowboat/app/scripts/rag_files_worker.ts

@@ -4,14 +4,29 @@ import { z } from 'zod';
 import { dataSourceDocsCollection, dataSourcesCollection } from '../lib/mongodb';
 import { EmbeddingRecord, DataSourceDoc, DataSource } from "../lib/types/datasource_types";
 import { WithId } from 'mongodb';
-import { embedMany } from 'ai';
+import { embedMany, generateText } from 'ai';
 import { embeddingModel } from '../lib/embedding';
 import { qdrantClient } from '../lib/qdrant';
 import { PrefixLogger } from "../lib/utils";
 import { GoogleGenerativeAI } from "@google/generative-ai";
 import { GetObjectCommand } from "@aws-sdk/client-s3";
 import { uploadsS3Client } from '../lib/uploads_s3_client';
+import fs from 'fs/promises';
 import crypto from 'crypto';
+import path from 'path';
+import { createOpenAI } from '@ai-sdk/openai';
+import { USE_GEMINI_FILE_PARSING } from '../lib/feature_flags';
+
+const FILE_PARSING_PROVIDER_API_KEY = process.env.FILE_PARSING_PROVIDER_API_KEY || process.env.OPENAI_API_KEY || '';
+const FILE_PARSING_PROVIDER_BASE_URL = process.env.FILE_PARSING_PROVIDER_BASE_URL || undefined;
+const FILE_PARSING_MODEL = process.env.FILE_PARSING_MODEL || 'gpt-4o';
+
+const openai = createOpenAI({
+    apiKey: FILE_PARSING_PROVIDER_API_KEY,
+    baseURL: FILE_PARSING_PROVIDER_BASE_URL,
+});
+
+const UPLOADS_DIR = process.env.RAG_UPLOADS_DIR || '/uploads';
 
 const splitter = new RecursiveCharacterTextSplitter({
     separators: ['\n\n', '\n', '. ', '.', ''],
@@ -27,7 +42,11 @@ const day = 24 * hour;
 // Configure Google Gemini API
 const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY || '');
 
-async function getFileContent(s3Key: string): Promise<Buffer> {
+async function getLocalFileContent(path: string): Promise<Buffer> {
+    return await fs.readFile(path);
+}
+
+async function getS3FileContent(s3Key: string): Promise<Buffer> {
     const command = new GetObjectCommand({
         Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
         Key: s3Key,
@@ -54,33 +73,59 @@ async function retryable<T>(fn: () => Promise<T>, maxAttempts: number = 3): Prom
     }
 }
 
-async function runProcessPipeline(_logger: PrefixLogger, job: WithId<z.infer<typeof DataSource>>, doc: WithId<z.infer<typeof DataSourceDoc>>): Promise<void> {
+async function runProcessPipeline(_logger: PrefixLogger, job: WithId<z.infer<typeof DataSource>>, doc: WithId<z.infer<typeof DataSourceDoc>> & { data: { type: "file_local" | "file_s3" } }): Promise<void> {
     const logger = _logger
         .child(doc._id.toString())
         .child(doc.name);
 
-    // Get file content from S3
-    logger.log("Fetching file from S3");
-    if (doc.data.type !== 'file') {
-        throw new Error("Invalid data source type");
+    // Get file content
+    let fileData: Buffer;
+    if (doc.data.type === 'file_local') {
+        logger.log("Fetching file from local");
+        fileData = await getLocalFileContent(path.join(UPLOADS_DIR, doc._id.toString()));
+    } else {
+        logger.log("Fetching file from S3");
+        fileData = await getS3FileContent(doc.data.s3Key);
     }
-    const fileData = await getFileContent(doc.data.s3Key);
 
-    // Use Gemini to extract text content
-    logger.log("Extracting content using Gemini");
-    const model = genAI.getGenerativeModel({ model: "gemini-2.0-flash-001" });
-    const prompt = "Extract and return only the text content from this document in markdown format. Exclude any formatting instructions or additional commentary.";
-    
-    const result = await model.generateContent([
-        {
-            inlineData: {
-                data: fileData.toString('base64'),
-                mimeType: doc.data.mimeType
-            }
-        },
-        prompt
-    ]);
-    const markdown = result.response.text();
+    let markdown = "";
+    const extractPrompt = "Extract and return only the text content from this document in markdown format. Exclude any formatting instructions or additional commentary.";
+    if (!USE_GEMINI_FILE_PARSING) {
+        // Use OpenAI to extract text content
+        logger.log("Extracting content using OpenAI");
+        const { text } = await generateText({
+            model: openai(FILE_PARSING_MODEL),
+            system: extractPrompt,
+            messages: [
+                {
+                    role: "user",
+                    content: [
+                        {
+                            type: "file",
+                            data: fileData.toString('base64'),
+                            mimeType: doc.data.mimeType,
+                        }
+                    ]
+                }
+            ],
+        });
+        markdown = text;
+    } else {
+        // Use Gemini to extract text content
+        logger.log("Extracting content using Gemini");
+        const model = genAI.getGenerativeModel({ model: "gemini-2.0-flash-001" });
+
+        const result = await model.generateContent([
+            {
+                inlineData: {
+                    data: fileData.toString('base64'),
+                    mimeType: doc.data.mimeType
+                }
+            },
+            extractPrompt,
+        ]);
+        markdown = result.response.text();
+    }
 
     // split into chunks
     logger.log("Splitting into chunks");
@@ -165,13 +210,13 @@ async function runDeletionPipeline(_logger: PrefixLogger, job: WithId<z.infer<ty
 // fetch next job from mongodb
 (async () => {
     while (true) {
-        console.log("Polling for job...")
         const now = Date.now();
         let job: WithId<z.infer<typeof DataSource>> | null = null;
 
         // first try to find a job that needs deleting
         job = await dataSourcesCollection.findOneAndUpdate({
             status: "deleted",
+            "data.type": { $in: ["files_local", "files_s3"] },
             $or: [
                 { attempts: { $exists: false } },
                 { attempts: { $lte: 3 } }
@@ -183,7 +228,7 @@ async function runDeletionPipeline(_logger: PrefixLogger, job: WithId<z.infer<ty
             job = await dataSourcesCollection.findOneAndUpdate(
                 {
                     $and: [
-                        { 'data.type': { $eq: "files" } },
+                        { 'data.type': { $in: ["files_local", "files_s3"] } },
                         {
                             $or: [
                                 // if the job has never been attempted
@@ -234,7 +279,7 @@ async function runDeletionPipeline(_logger: PrefixLogger, job: WithId<z.infer<ty
         let errors = false;
 
         try {
-            if (job.data.type !== 'files') {
+            if (job.data.type !== 'files_local' && job.data.type !== 'files_s3') {
                 throw new Error("Invalid data source type");
             }
 
@@ -276,8 +321,9 @@ async function runDeletionPipeline(_logger: PrefixLogger, job: WithId<z.infer<ty
 
             // for each doc
             for (const doc of pendingDocs) {
+                const ldoc = doc as WithId<z.infer<typeof DataSourceDoc>> & { data: { type: "file_local" | "file_s3" } };
                 try {
-                    await runProcessPipeline(logger, job, doc);
+                    await runProcessPipeline(logger, job, ldoc);
                 } catch (e: any) {
                     errors = true;
                     logger.log("Error processing doc:", e);

apps/rowboat/app/scripts/rag_text_worker.ts

@@ -112,13 +112,13 @@ async function runDeletionPipeline(_logger: PrefixLogger, job: WithId<z.infer<ty
 // fetch next job from mongodb
 (async () => {
     while (true) {
-        console.log("Polling for job...")
         const now = Date.now();
         let job: WithId<z.infer<typeof DataSource>> | null = null;
 
         // first try to find a job that needs deleting
         job = await dataSourcesCollection.findOneAndUpdate({
             status: "deleted",
+            "data.type": "text",
             $or: [
                 { attempts: { $exists: false } },
                 { attempts: { $lte: 3 } }

apps/rowboat/app/scripts/rag_urls_worker.ts

@@ -143,13 +143,13 @@ async function runDeletionPipeline(_logger: PrefixLogger, job: WithId<z.infer<ty
 // fetch next job from mongodb
 (async () => {
     while (true) {
-        console.log("Polling for job...")
         const now = Date.now();
         let job: WithId<z.infer<typeof DataSource>> | null = null;
 
         // first try to find a job that needs deleting
         job = await dataSourcesCollection.findOneAndUpdate({
             status: "deleted",
+            "data.type": "urls",
             $or: [
                 { attempts: { $exists: false } },
                 { attempts: { $lte: 3 } }

apps/rowboat/app/scripts/setup_qdrant.ts

@@ -1,14 +1,18 @@
 import '../lib/loadenv';
 import { qdrantClient } from '../lib/qdrant';
 
-(async () => {
-    await qdrantClient.createCollection('embeddings', {
-        vectors: {
-            size: 1536,
-            distance: 'Dot',
-        },
-    });
+const EMBEDDING_VECTOR_SIZE = Number(process.env.EMBEDDING_VECTOR_SIZE) || 1536;
 
-    const { collections } = await qdrantClient.getCollections();
-    console.log(collections);
+(async () => {
+    try {
+        const result = await qdrantClient.createCollection('embeddings', {
+            vectors: {
+                size: EMBEDDING_VECTOR_SIZE,
+                distance: 'Dot',
+            },
+        });
+        console.log(`Create qdrant collection 'embeddings' completed with result: ${result}`);
+    } catch (error) {
+        console.error(`Unable to create qdrant collection 'embeddings': ${error}`);
+    }
 })();

apps/rowboat/components/common/product-tour.tsx

@@ -12,42 +12,47 @@ const TOUR_STEPS: TourStep[] = [
     {
         target: 'copilot',
         content: 'Build agents with the help of copilot.\nThis might take a minute.',
-        title: 'Step 1/8'
+        title: 'Step 1/9'
     },
     {
         target: 'playground',
         content: 'Test your assistant in the playground.\nDebug tool calls and responses.',
-        title: 'Step 2/8'
+        title: 'Step 2/9'
     },
     {
         target: 'entity-agents',
         content: 'Manage your agents.\nSpecify instructions, examples and tool usage.',
-        title: 'Step 3/8'
+        title: 'Step 3/9'
     },
     {
         target: 'entity-tools',
         content: 'Create your own tools, import MCP tools or use existing ones.\nMock tools for quick testing.',
-        title: 'Step 4/8'
+        title: 'Step 4/9'
     },
     {
         target: 'entity-prompts',
         content: 'Manage prompts which will be used by agents.\nConfigure greeting message.',
-        title: 'Step 5/8'
+        title: 'Step 5/9'
+    },
+    {
+        target: 'entity-data-sources',
+        content: 'Add and manage RAG data sources which will be used by agents.\nAvailable sources are local files, S3 files, web URLs and plain text.  \n\nIMPORTANT: Once you have added a data source, make sure to add it inside your\nagent configuration and agent instructions (mention the @tool:rag_search).',
+        title: 'Step 6/9'
     },
     {
         target: 'settings',
         content: 'Configure project settings\nGet API keys, configure tool webhooks.',
-        title: 'Step 6/8'
+        title: 'Step 7/9'
     },
     {
         target: 'deploy',
         content: 'Deploy your workflow version to make it live.\nThis will make your workflow available for use via the API and SDK.\n\nLearn more:\n• <a href="https://docs.rowboatlabs.com/using_the_api/" target="_blank" class="text-indigo-600 hover:text-indigo-700 dark:text-indigo-400 dark:hover:text-indigo-300">Using the API</a>\n• <a href="https://docs.rowboatlabs.com/using_the_sdk/" target="_blank" class="text-indigo-600 hover:text-indigo-700 dark:text-indigo-400 dark:hover:text-indigo-300">Using the SDK</a>',
-        title: 'Step 7/8'
+        title: 'Step 8/9'
     },
     {
         target: 'tour-button',
         content: 'Come back here anytime to restart the tour.\nStill have questions? See our <a href="https://docs.rowboatlabs.com/" target="_blank" class="text-indigo-600 hover:text-indigo-700 dark:text-indigo-400 dark:hover:text-indigo-300">docs</a> or reach out on <a href="https://discord.gg/gtbGcqF4" target="_blank" class="text-indigo-600 hover:text-indigo-700 dark:text-indigo-400 dark:hover:text-indigo-300">discord</a>.',
-        title: 'Step 8/8'
+        title: 'Step 9/9'
     }
 ];
 

apps/rowboat/package.json

@@ -15,7 +15,7 @@
     "ragTextWorker": "tsx app/scripts/rag_text_worker.ts"
   },
   "dependencies": {
-    "@ai-sdk/openai": "^0.0.37",
+    "@ai-sdk/openai": "^1.3.21",
     "@auth0/nextjs-auth0": "^3.5.0",
     "@aws-sdk/client-s3": "^3.743.0",
     "@aws-sdk/s3-request-presigner": "^3.743.0",
@@ -31,7 +31,7 @@
     "@modelcontextprotocol/sdk": "^1.7.0",
     "@primer/react": "^36.27.0",
     "@qdrant/js-client-rest": "^1.13.0",
-    "ai": "^3.3.28",
+    "ai": "^4.3.13",
     "cheerio": "^1.0.0",
     "class-variance-authority": "^0.7.1",
     "clsx": "^2.1.1",

apps/rowboat_agents/src/app/main.py

@@ -16,10 +16,8 @@ print("Master config:", master_config)
 
 # Get environment variables with defaults
 ENABLE_TRACING = False
-try:
+if os.environ.get('ENABLE_TRACING'):
     ENABLE_TRACING = os.environ.get('ENABLE_TRACING').lower() == 'true'
-except Exception as e:
-    print(f"Error getting ENABLE_TRACING: {e}, using default of False")
 
 # filter out agent transfer messages using a function
 def is_agent_transfer_message(msg):

apps/rowboat_agents/src/graph/execute_turn.py

@@ -153,7 +153,7 @@ def get_rag_tool(config: dict, complete_request: dict) -> FunctionTool:
     """
     project_id = complete_request.get("projectId", "")
     if config.get("ragDataSources", None):
-        print("getArticleInfo")
+        print(f"Creating rag_search tool with params:\n-Data Sources: {config.get('ragDataSources', [])}\n-Return Type: {config.get('ragReturnType', 'chunks')}\n-K: {config.get('ragK', 3)}")
         params = {
             "type": "object",
             "properties": {
@@ -168,10 +168,10 @@ def get_rag_tool(config: dict, complete_request: dict) -> FunctionTool:
             ]
         }
         tool = FunctionTool(
-            name="getArticleInfo",
-            description="Get information about an article",
-            params_json_schema=params,
-            on_invoke_tool=lambda ctx, args: call_rag_tool(project_id, json.loads(args)['query'], config.get("ragDataSources", []), "chunks", 3)
+                name="rag_search",
+                description="Get information about an article",
+                params_json_schema=params,
+                on_invoke_tool=lambda ctx, args: call_rag_tool(project_id, json.loads(args)['query'], config.get("ragDataSources", []), config.get("ragReturnType", "chunks"), config.get("ragK", 3))
         )
         return tool
     else:
@@ -208,10 +208,6 @@ def get_agents(agent_configs, tool_configs, complete_request):
         print(f"Agent {agent_config['name']} has {len(agent_config['tools'])} configured tools")
 
         new_tools = []
-        rag_tool = get_rag_tool(agent_config, complete_request)
-        if rag_tool:
-            new_tools.append(rag_tool)
-            print(f"Added rag tool to agent {agent_config['name']}")
 
         for tool_name in agent_config["tools"]:
 
@@ -224,6 +220,10 @@ def get_agents(agent_configs, tool_configs, complete_request):
                 })
                 if tool_name == "web_search":
                     tool = WebSearchTool()
+
+                elif tool_name == "rag_search":
+                    tool = get_rag_tool(agent_config, complete_request)
+
                 else:
                     tool = FunctionTool(
                         name=tool_name,
@@ -233,8 +233,9 @@ def get_agents(agent_configs, tool_configs, complete_request):
                     on_invoke_tool=lambda ctx, args, _tool_name=tool_name, _tool_config=tool_config, _complete_request=complete_request:
                         catch_all(ctx, args, _tool_name, _tool_config, _complete_request)
                     )
-                new_tools.append(tool)
-                print(f"Added tool {tool_name} to agent {agent_config['name']}")
+                if tool:
+                    new_tools.append(tool)
+                    print(f"Added tool {tool_name} to agent {agent_config['name']}")
             else:
                 print(f"WARNING: Tool {tool_name} not found in tool_configs")
 

apps/rowboat_agents/src/graph/tool_calling.py

@@ -75,18 +75,19 @@ async def call_rag_tool(
         "active": True
     }).to_list(length=None)
 
-    print(sources)
+    print(f"Sources: {sources}")
     # Filter sources to those in source_ids
     valid_source_ids = [
         str(s["_id"]) for s in sources if str(s["_id"]) in source_ids
     ]
 
-    print(valid_source_ids)
+    print(f"Valid source ids: {valid_source_ids}")
     # If no valid sources are found, return empty results
     if not valid_source_ids:
         return ''
 
     # Perform Qdrant vector search
+    print(f"Calling Qdrant search with limit {k}")
     qdrant_results = qdrant_client.search(
         collection_name="embeddings",
         query_vector=embed_result["embedding"],
@@ -112,11 +113,13 @@ async def call_rag_tool(
         for point in qdrant_results
     ]
 
-    print(return_type)
-    print(results)
+    print(f"Return type: {return_type}")
+    print(f"Results: {results}")
     # If return_type is 'chunks', return the results directly
     if return_type == "chunks":
-        return json.dumps({"Information": results}, indent=2)
+        chunks = json.dumps({"Information": results}, indent=2)
+        print(f"Returning chunks: {chunks}")
+        return chunks
 
     # Otherwise, fetch the full document contents from MongoDB
     doc_ids = [ObjectId(r["docId"]) for r in results]
@@ -132,10 +135,9 @@ async def call_rag_tool(
     ]
 
     # Convert results to a JSON string
-    formatted_string = json.dumps({"Information": results}, indent=2)
-    print(formatted_string)
-    return formatted_string
-
+    docs = json.dumps({"Information": results}, indent=2)
+    print(f"Returning docs: {docs}")
+    return docs
 
 if __name__ == "__main__":
     asyncio.run(call_rag_tool(

apps/rowboat_agents/src/graph/tools.py

@@ -1,43 +1,8 @@
 import json
 import random
 
-from src.utils.common import common_logger
-logger = common_logger
-
-RAG_TOOL = {
-    "name": "getArticleInfo",
-    "type": "rag",
-    "description": "Fetch articles with knowledge relevant to the query",
-    "parameters": {
-        "type": "object",
-        "properties": {
-            "question": {
-                "type": "string",
-                "description": "The query to retrieve articles for"
-            }
-        },
-        "required": [
-            "query"
-        ]
-    }
-}
-
-CLOSE_CHAT_TOOL = {
-    "name": "close_chat",
-    "type": "close_chat",
-    "description": "Close the chat",
-    "parameters": {
-        "type": "object",
-        "properties": {
-            "error_message": {
-                "type": "string", "description": "The error message to close the chat with"
-            }
-        }
-    }
-}
-
 def tool_raise_error(error_message):
-    logger.error(f"Raising error: {error_message}")
+    print(f"Raising error: {error_message}")
     raise ValueError(f"Raising error: {error_message}")
 
 def respond_to_tool_raise_error(tool_calls, mock=False):
@@ -45,7 +10,7 @@ def respond_to_tool_raise_error(tool_calls, mock=False):
     return _create_tool_response(tool_calls, tool_raise_error(error_message))
 
 def tool_close_chat(error_message):
-    logger.error(f"Closing chat: {error_message}")
+    print(f"Closing chat: {error_message}")
     raise ValueError(f"Closing chat: {error_message}")
 
 def respond_to_tool_close_chat(tool_calls, mock=False):

apps/rowboat_agents/tests/interactive.py

@@ -3,9 +3,9 @@ from datetime import datetime
 import json
 import sys
 import asyncio
+import requests
+import argparse
 
-from src.graph.core import order_messages, run_turn_streamed
-from src.graph.tools import respond_to_tool_raise_error, respond_to_tool_close_chat, RAG_TOOL, CLOSE_CHAT_TOOL
 from src.utils.common import common_logger, read_json_from_file
 logger = common_logger
 
@@ -26,103 +26,144 @@ def preprocess_messages(messages):
             msg["role"] = "user"
     
     return messages
-        
-async def process_turn(messages, agent_configs, tool_configs, prompt_configs, start_agent_name, state, config, complete_request):
-    """Processes a single turn using streaming API"""
-    print(f"\n{'*'*50}\nLatest Request:\n{'*'*50}")
-    request_json = {
-        "messages": [{k: v for k, v in msg.items() if k != 'current_turn'} for msg in messages],
-        "state": state,
-        "agents": agent_configs,
-        "tools": tool_configs,
-        "prompts": prompt_configs,
-        "startAgent": start_agent_name
-    }
-    print(json.dumps(request_json, indent=2))
 
-    collected_messages = []
+def stream_chat(host, request_data, api_key):
+    start_time = datetime.now()
+    print("\n" + "="*80)
+    print(f"Starting streaming chat at {start_time}")
+    print(f"Host: {host}")
+    print("="*80 + "\n")
+
+    try:
+        print("\n" + "-"*80)
+        print("Connecting to stream...")
+        stream_response = requests.post(
+            f"{host}/chat_stream",
+            json=request_data,
+            headers={
+                'Authorization': f'Bearer {api_key}',
+                'Accept': 'text/event-stream'
+            },
+            stream=True
+        )
+        
+        if stream_response.status_code != 200:
+            print(f"Error connecting to stream. Status code: {stream_response.status_code}")
+            print(f"Response: {stream_response.text}")
+            return None, None
+            
+        print(f"Successfully connected to stream")
+        print("-"*80 + "\n")
+
+        event_count = 0
+        collected_messages = []
+        final_state = None
+        
+        try:
+            print("\n" + "-"*80)
+            print("Starting to process events...")
+            print("-"*80 + "\n")
+            
+            for line in stream_response.iter_lines(decode_unicode=True):
+                if line:
+                    if line.startswith('data: '):
+                        data = line[6:]  # Remove 'data: ' prefix
+                        try:
+                            event_data = json.loads(data)
+                            event_count += 1
+                            print("\n" + "*"*80)
+                            print(f"Event #{event_count} at {datetime.now().isoformat()}")
+                            
+                            if isinstance(event_data, dict):
+                                # Pretty print the event data
+                                print("Event Data:")
+                                print(json.dumps(event_data, indent=2))
+                                
+                                # Special handling for message events
+                                if 'content' in event_data:
+                                    print("\nMessage Content:", event_data['content'])
+                                    if event_data.get('tool_calls'):
+                                        print("Tool Calls:", json.dumps(event_data['tool_calls'], indent=2))
+                                    
+                                    # Collect messages
+                                    collected_messages.append(event_data)
+                            else:
+                                print("Event Data:", event_data)
+                            print("*"*80 + "\n")
+                            
+                        except json.JSONDecodeError as e:
+                            print(f"Error decoding event data: {e}")
+                            print(f"Raw data: {data}")
+                
+        except Exception as e:
+            print(f"Error processing stream: {e}")
+            import traceback
+            traceback.print_exc()
+        finally:
+            print("\n" + "-"*80)
+            print(f"Closing stream after processing {event_count} events")
+            print("-"*80 + "\n")
+            stream_response.close()
+            
+    except requests.exceptions.RequestException as e:
+        print(f"Request error during streaming: {e}")
+        import traceback
+        traceback.print_exc()
+
+    end_time = datetime.now()
+    duration = end_time - start_time
+    print("\n" + "="*80)
+    print(f"Streaming session completed at {end_time}")
+    print(f"Total duration: {duration}")
+    print("="*80 + "\n")
     
-    async for event_type, event_data in run_turn_streamed(
-        messages=messages,
-        start_agent_name=start_agent_name,
-        agent_configs=agent_configs,
-        tool_configs=tool_configs,
-        prompt_configs=prompt_configs,
-        start_turn_with_start_agent=config.get("start_turn_with_start_agent", False),
-        state=state,
-        additional_tool_configs=[RAG_TOOL, CLOSE_CHAT_TOOL],
-        complete_request=complete_request
-    ):
-        if event_type == "message":
-            # Add each message to collected_messages
-            collected_messages.append(event_data)
-            
-        elif event_type == "done":
-            print(f"\n\n{'*'*50}\nLatest Response:\n{'*'*50}")
-            response_json = {
-                "messages": collected_messages,
-                "state": event_data.get('state', {}),
-            }
-            print("Turn completed. Here are the streamed messages and final state:")
-            print(json.dumps(response_json, indent=2))
-            print('='*50)
-            
-            return collected_messages, event_data.get('state', {})
-            
-        elif event_type == "error":
-            print(f"\nError: {event_data.get('error', 'Unknown error')}")
-            return [], state
+    return collected_messages, final_state
 
 if __name__ == "__main__":
     logger.info(f"{'*'*50}Running interactive mode{'*'*50}")
 
-    def extract_request_fields(complete_request):
-        agent_configs = complete_request.get("agents", [])
-        tool_configs = complete_request.get("tools", [])
-        prompt_configs = complete_request.get("prompts", [])
-        start_agent_name = complete_request.get("startAgent", "")
-        
-        return agent_configs, tool_configs, prompt_configs, start_agent_name
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', type=str, required=False, default='default_config.json',
+                       help='Config file name under configs/')
+    parser.add_argument('--sample_request', type=str, required=False, default='default_example.json',
+                       help='Sample request JSON file name under tests/sample_requests/')
+    parser.add_argument('--api_key', type=str, required=False, default='test',
+                       help='API key to use for authentication')
+    parser.add_argument('--host', type=str, default='http://localhost:4040',
+                       help='Host to use for the request')
+    parser.add_argument('--load_messages', action='store_true',
+                       help='Load messages from sample request file')
+    args = parser.parse_args()
+
+    print(f"Config file: {args.config}")
+    print(f"Sample request file: {args.sample_request}")
     
-    external_tool_mappings = {
-        "raise_error": respond_to_tool_raise_error,
-        "close_chat": respond_to_tool_close_chat
-    }
+    config = read_json_from_file(f"./configs/{args.config}")
+    example_request = read_json_from_file(f"./tests/sample_requests/{args.sample_request}").get("lastRequest", {})
     
-    config_file = sys.argv[sys.argv.index("--config") + 1] if "--config" in sys.argv else "default_config.json"
-    sample_request_file = sys.argv[sys.argv.index("--sample_request") + 1] if "--sample_request" in sys.argv else "default_example.json"
-    
-    print(f"Config file: {config_file}")
-    print(f"Sample request file: {sample_request_file}")
-    
-    config = read_json_from_file(f"./configs/{config_file}")
-    example_request = read_json_from_file(f"./tests/sample_requests/{sample_request_file}").get("lastRequest", {})
-    
-    if "--load_messages" in sys.argv:
+    if args.load_messages:
         messages = example_request.get("messages", [])
-        messages = order_messages(messages)
         user_input_needed = False
     else:
         messages = []
         user_input_needed = True
 
-    turn_start_time = datetime.now()
-    tool_duration = 0
-
     state = example_request.get("state", {})
     start_agent_name = example_request.get("startAgent", "")
     last_agent_name = state.get("last_agent_name", "")
     if not last_agent_name:
         last_agent_name = start_agent_name
 
-
     logger.info("Starting main conversation loop")
+    start_time = None
     while True:
         logger.info("Loading configuration files")
 
         # To account for updates to state
         complete_request = copy.deepcopy(example_request)
-        agent_configs, tool_configs, prompt_configs, start_agent_name = extract_request_fields(complete_request)
+        complete_request["messages"] = messages
+        complete_request["state"] = state
+        complete_request["startAgent"] = start_agent_name
 
         print(f"\nUsing agent: {last_agent_name}")
         
@@ -132,75 +173,35 @@ if __name__ == "__main__":
                 "role": "user",
                 "content": user_inp
             })
-            turn_start_time = datetime.now()
-            tool_duration = 0
             if user_inp == 'exit':
                 logger.info("User requested exit")
                 break
             logger.info("Added user message to conversation")
+        
+        start_time = datetime.now()
 
-        # Preprocess messages to replace role tool with role developer and add role user to empty roles
-        print("Preprocessing messages to replace role tool with role developer and add role user to empty roles")
+        # Preprocess messages
+        print("Preprocessing messages")
         messages = preprocess_messages(messages)
         complete_request["messages"] = preprocess_messages(complete_request["messages"])
 
         # Run the streaming turn
-        resp_messages, resp_state = asyncio.run(process_turn(
-            messages=messages,
-            agent_configs=agent_configs,
-            tool_configs=tool_configs,
-            prompt_configs=prompt_configs,
-            start_agent_name=start_agent_name,
-            state=state,
-            config=config,
-            complete_request=complete_request
-        ))
+        resp_messages, resp_state = stream_chat(
+            host=args.host,
+            request_data=complete_request,
+            api_key=args.api_key
+        )
         
-        state = resp_state
-        last_msg = resp_messages[-1] if resp_messages else {}
-        tool_calls = last_msg.get("tool_calls", [])
-        sender = last_msg.get("sender", "")
-            
-        if config.get("return_diff_messages", True):
-            messages.extend(resp_messages)
-        else:
-            messages = resp_messages
+        if resp_messages:
+            state = resp_state
+            if config.get("return_diff_messages", True):
+                messages.extend(resp_messages)
+            else:
+                messages = resp_messages
 
-        if tool_calls:
-            tool_start_time = datetime.now()
-            user_input_needed = False
-
-            should_break = False
-            for tool_call in tool_calls:
-                tool_name = tool_call["function"]["name"]
-                logger.info(f"Processing tool call: {tool_name}")
-
-                if tool_name not in external_tool_mappings:
-                    logger.error(f"Unknown tool call: {tool_name}")
-                    raise ValueError(f"Unknown tool call: {tool_name}")
-
-                # Call appropriate handler and process response
-                tool_response = external_tool_mappings[tool_name]([tool_call], mock=True)
-                messages.append(tool_response)
-                logger.info(f"Added {tool_name} response to messages")
-
-                current_tool_duration = round((datetime.now() - tool_start_time).total_seconds() * 10) / 10
-                logger.info(f"Tool response duration: {current_tool_duration:.1f}s")
-                tool_duration += current_tool_duration
-                
-                if tool_name == "close_chat":
-                    user_input_needed = False
-                    logger.info("Closing chat")
-                    should_break = True
-
-            if should_break:
-                break
-        
-        else:
-            user_input_needed = True
-            print("Quick stats")
-            print(f"Turn Duration: {round((datetime.now() - turn_start_time).total_seconds() * 10) / 10:.1f}s")
-            print(f"Tool Response Duration: {round(tool_duration * 10) / 10:.1f}s")
-            print('='*50)
+        user_input_needed = True
+        print("Quick stats")
+        print(f"Turn Duration: {round((datetime.now() - start_time).total_seconds() * 10) / 10:.1f}s")
+        print('='*50)
             
     print("\n" + "-" * 80)

apps/rowboat_agents/tests/sample_requests/tmp4.json

@@ -7,22 +7,58 @@
     {
       "version": "v1",
       "chatId": "",
-      "createdAt": "2025-05-08T12:18:10.212Z",
+      "createdAt": "2025-05-08T07:06:48.280Z",
       "role": "assistant",
       "content": "How can I help you today?",
-      "agenticSender": "Blog Writer Hub",
+      "agenticSender": "Example Agent",
       "agenticResponseType": "external"
     },
     {
       "role": "user",
-      "content": "write a blog on bitcoin",
+      "content": "Tell me about article 1",
       "version": "v1",
       "chatId": "",
-      "createdAt": "2025-05-08T12:18:15.023Z"
+      "createdAt": "2025-05-08T07:06:53.518Z"
+    },
+    {
+      "version": "v1",
+      "chatId": "",
+      "createdAt": "2025-05-08T07:06:55.690Z",
+      "role": "assistant",
+      "tool_calls": [
+        {
+          "id": "call_uzRd4Y1CeBioJ9h26XeGhCvH",
+          "function": {
+            "name": "rag_search",
+            "arguments": "{\"query\":\"article 1\"}"
+          },
+          "type": "function"
+        }
+      ],
+      "agenticSender": "Example Agent",
+      "agenticResponseType": "internal"
+    },
+    {
+      "version": "v1",
+      "chatId": "",
+      "createdAt": "2025-05-08T07:06:55.690Z",
+      "role": "tool",
+      "content": "{\n  \"Information\": [\n    {\n      \"title\": \"text\",\n      \"name\": \"text\",\n      \"content\": \"This is article 1.\",\n      \"docId\": \"681c50f17e7a18621c7215cd\",\n      \"sourceId\": \"681c50f17e7a18621c7215cc\"\n    },\n    {\n      \"title\": \"text\",\n      \"name\": \"text\",\n      \"content\": \"This is article 2.\",\n      \"docId\": \"681c51067e7a18621c7215cf\",\n      \"sourceId\": \"681c51067e7a18621c7215ce\"\n    },\n    {\n      \"title\": \"text\",\n      \"name\": \"text\",\n      \"content\": \"This is article 3.\",\n      \"docId\": \"681c51137e7a18621c7215d1\",\n      \"sourceId\": \"681c51137e7a18621c7215d0\"\n    }\n  ]\n}",
+      "tool_call_id": "call_uzRd4Y1CeBioJ9h26XeGhCvH",
+      "tool_name": ""
+    },
+    {
+      "version": "v1",
+      "chatId": "",
+      "createdAt": "2025-05-08T07:06:57.096Z",
+      "role": "assistant",
+      "content": "Article 1 states: \"This is article 1.\"\n\nIf you need more details or information about another article, please let me know!",
+      "agenticSender": "Example Agent",
+      "agenticResponseType": "external"
     }
   ],
   "lastRequest": {
-    "projectId": "e088bfb0-793d-4da8-8e3d-001bfb6fb647",
+    "projectId": "8e59c4b6-91d2-42e9-8990-41583f4105f1",
     "messages": [
       {
         "content": "",
@@ -35,14 +71,14 @@
       {
         "content": "How can I help you today?",
         "role": "assistant",
-        "sender": "Blog Writer Hub",
+        "sender": "Example Agent",
         "tool_calls": null,
         "tool_call_id": null,
         "tool_name": null,
         "response_type": "external"
       },
       {
-        "content": "write a blog on bitcoin",
+        "content": "Tell me about article 1",
         "role": "user",
         "sender": null,
         "tool_calls": null,
@@ -51,84 +87,31 @@
       }
     ],
     "state": {
-      "last_agent_name": "Blog Writer Hub",
+      "last_agent_name": "Example Agent",
       "tokens": {
         "total": 0,
         "prompt": 0,
         "completion": 0
-      },
-      "turn_messages": [
-        {
-          "content": "How can I help you today?",
-          "role": "assistant",
-          "sender": "Blog Writer Hub",
-          "tool_calls": null,
-          "tool_call_id": null,
-          "tool_name": null,
-          "response_type": "external"
-        }
-      ]
+      }
     },
     "agents": [
       {
-        "name": "Blog Writer Hub",
+        "name": "Example Agent",
         "type": "conversation",
-        "description": "Hub agent to orchestrate the blog writing process from research to final blog post.",
-        "instructions": "## 🧑‍💼 Role:\nYou are the hub agent responsible for orchestrating the blog writing process for the user.\n\n---\n## ⚙️ Steps to Follow:\n1. Greet the user and ask for the blog topic.\n2. FIRST: Send the topic to [@agent:Research Agent] to gather and compile research notes.\n3. Wait for the complete research notes from the Research Agent.\n4. THEN: Send the research notes to [@agent:Outline Agent] to generate a detailed outline.\n5. Wait for the outline from the Outline Agent.\n6. THEN: Send the outline to [@agent:Writing Agent] to write the full blog post (1000+ words).\n7. Once the blog post is ready, return it to the user.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Orchestrating the sequential process of research, outlining, and writing\n- Returning the final blog post to the user\n\n❌ Out of Scope:\n- Performing research, outlining, or writing directly\n- Handling requests unrelated to blog writing\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Follow the strict sequence: Research → Outline → Writing\n- Wait for each agent's complete response before proceeding\n- Only interact with the user for topic input and final blog post delivery\n\n🚫 Don'ts:\n- Do not perform research, outlining, or writing yourself\n- Do not try to get responses from multiple agents simultaneously\n- CRITICAL: The system does not support more than 1 tool call in a single output when the tool call is about transferring to another agent (a handoff). You must only put out 1 transfer related tool call in one output.\n\n# Examples\n- **User** : I want a blog post about 'Benefits of Remote Work'.\n - **Agent actions**: Call [@agent:Research Agent]\n\n- **Agent receives research notes** :\n - **Agent actions**: Call [@agent:Outline Agent]\n\n- **Agent receives outline** :\n - **Agent actions**: Call [@agent:Writing Agent]\n\n- **Agent receives blog post** :\n - **Agent response**: Here is your completed blog post on 'Benefits of Remote Work': [Full blog post]\n\n- **User** : Can you write a blog post on 'AI in Healthcare'?\n - **Agent actions**: Call [@agent:Research Agent](#mention)\n\n- **User** : Hi!\n - **Agent response**: Hello! What blog topic would you like to write about today?",
+        "description": "An example agent that uses the rag_search tool to fetch information for customer support queries.",
+        "instructions": "## 🧑‍💼 Role:\nYou are a helpful customer support assistant who fetches information using the rag_search tool.\n\n---\n## ⚙️ Steps to Follow:\n1. Ask the user what they would like help with.\n2. If the user's query requires information or an answer, use the [@tool:rag_search] tool with the user's question as the query.\n3. Provide the user with an answer based on the information retrieved from the tool.\n4. If the user's issue is about follow-up or requires human support, ask for their email address and let them know someone will contact them soon.\n5. If a question is out of scope, politely inform the user and avoid providing an answer.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Answering customer support questions using information from articles.\n- Asking the user their issue and getting their email if needed.\n\n❌ Out of Scope:\n- Questions unrelated to customer support.\n- Providing answers without using the rag_search tool when information is needed.\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Use [@tool:rag_search](#mention) to fetch accurate information for user queries.\n- Provide clear and concise answers based on the tool's results.\n- Ask for the user's email if the issue requires follow-up.\n\n🚫 Don'ts:\n- Do not provide answers without fetching information when required.\n- Do not ask for details other than the user's email.\n\n# Examples\n- **User** : What is your refund policy?\n - **Agent actions**: Call [@tool:rag_search](#mention)\n - **Agent response**: Our refund policy is as follows: <summary from rag_search results>\n\n- **User** : How can I change my password?\n - **Agent actions**: Call [@tool:rag_search](#mention)\n - **Agent response**: To change your password, follow these steps: <steps from rag_search results>\n\n- **User** : I need help with my order.\n - **Agent actions**: Call [@tool:rag_search](#mention)\n - **Agent response**: Can you please provide more details about your order issue? <additional info from rag_search results if relevant>\n\n- **User** : I want someone to contact me about a billing issue.\n - **Agent response**: Sure, could you please provide your email address so someone from our team can contact you soon?\n\n- **User** : Can you tell me about your subscription plans?\n - **Agent actions**: Call [@tool:rag_search](#mention)\n - **Agent response**: Here are our subscription plans: <details from rag_search results>",
         "model": "gpt-4.1",
         "controlType": "retain",
-        "ragK": 3,
-        "ragReturnType": "chunks",
-        "outputVisibility": "user_facing",
-        "tools": [],
-        "prompts": [],
-        "connectedAgents": [
-          "Research Agent",
-          "Outline Agent",
-          "Writing Agent"
-        ]
-      },
-      {
-        "name": "Research Agent",
-        "type": "conversation",
-        "description": "Researches the given blog topic and compiles relevant information.",
-        "instructions": "## 🧑‍💼 Role:\nYou are responsible for researching the provided blog topic and compiling relevant, up-to-date information.\n\n---\n## ⚙️ Steps to Follow:\n1. Use the [@tool:web_search] tool to gather information about the topic.\n2. Summarize and compile the most important and recent findings, statistics, and facts relevant to the topic.\n3. Return a concise, well-organized compilation of research notes (not an outline or blog post).\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Researching the topic using web sources\n- Compiling factual, relevant information\n\n❌ Out of Scope:\n- Creating outlines or writing the blog post\n- Providing opinions or unverified information\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Use multiple reputable sources\n- Organize findings clearly\n\n🚫 Don'ts:\n- Do not create outlines or write the blog post\n- Do not include personal opinions\n\n# Examples\n- **User** : Research the topic 'Benefits of Remote Work'.\n - **Agent actions**: Call [@tool:web_search](#mention)\n - **Agent response**: \"Compiled research notes: 1. Increased productivity: Studies show remote workers are 13% more productive (Stanford, 2020). 2. Cost savings: Companies save on office space and utilities. 3. Improved work-life balance...\"",
-        "model": "gpt-4.1",
-        "controlType": "retain",
-        "ragK": 3,
-        "ragReturnType": "chunks",
-        "outputVisibility": "internal",
-        "tools": [
-          "web_search"
+        "ragDataSources": [
+          "681c50f17e7a18621c7215cc",
+          "681c51067e7a18621c7215ce",
+          "681c51137e7a18621c7215d0"
         ],
-        "prompts": [],
-        "connectedAgents": []
-      },
-      {
-        "name": "Outline Agent",
-        "type": "conversation",
-        "description": "Creates a detailed bullet-point outline for the blog post based on research notes.",
-        "instructions": "## 🧑‍💼 Role:\nYou are responsible for creating a detailed outline for the blog post using the compiled research notes.\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the compiled research notes.\n2. Create a logical, well-structured bullet-point outline for the blog post (including introduction, main sections, and conclusion).\n3. Ensure the outline covers all key points from the research.\n4. Return only the outline (not the full blog post).\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Creating outlines for blog posts\n\n❌ Out of Scope:\n- Conducting research\n- Writing the full blog post\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Ensure logical flow and coverage of all research points\n- Use clear, concise bullet points\n\n🚫 Don'ts:\n- Do not write full paragraphs or the blog post itself\n\n# Examples\n- **User** : Create an outline for a blog on 'Benefits of Remote Work'.\n - **Agent response**: \"Outline: 1. Introduction 2. Increased Productivity 3. Cost Savings for Companies 4. Improved Work-Life Balance 5. Challenges and Solutions 6. Conclusion\"",
-        "model": "gpt-4.1",
-        "controlType": "retain",
-        "ragK": 3,
+        "ragK": 1,
         "ragReturnType": "chunks",
-        "outputVisibility": "internal",
-        "tools": [],
-        "prompts": [],
-        "connectedAgents": []
-      },
-      {
-        "name": "Writing Agent",
-        "type": "conversation",
-        "description": "Writes a full blog post (1000+ words) based on the provided outline.",
-        "instructions": "## 🧑‍💼 Role:\nYou are responsible for writing a complete, well-structured blog post based on the provided outline.\n\n---\n## ⚙️ Steps to Follow:\n1. Receive the outline for the blog post.\n2. Write a detailed, engaging blog post that follows the outline and incorporates all key points.\n3. Ensure the blog post is at least 1000 words.\n4. Return only the completed blog post.\n\n---\n## 🎯 Scope:\n✅ In Scope:\n- Writing full blog posts based on outlines\n\n❌ Out of Scope:\n- Creating outlines or conducting research\n- Writing posts shorter than 1000 words\n\n---\n## 📋 Guidelines:\n✔️ Dos:\n- Ensure clarity, coherence, and logical flow\n- Expand on each outline point with relevant details\n- Use engaging and professional language\n\n🚫 Don'ts:\n- Do not skip any outline sections\n- Do not write less than 1000 words\n\n# Examples\n- **User** : Write a blog post on 'Benefits of Remote Work' using the provided outline.\n - **Agent response**: \"[Full blog post, 1000+ words, covering all outline points]\"",
-        "model": "gpt-4.1",
-        "controlType": "retain",
-        "ragK": 3,
-        "ragReturnType": "chunks",
-        "outputVisibility": "internal",
-        "tools": [],
+        "tools": [
+          "rag_search"
+        ],
         "prompts": [],
         "connectedAgents": []
       }
@@ -142,12 +125,76 @@
           "properties": {}
         },
         "isLibrary": true
+      },
+      {
+        "name": "rag_search",
+        "description": "Fetch articles with knowledge relevant to the query",
+        "parameters": {
+          "type": "object",
+          "properties": {
+            "query": {
+              "type": "string",
+              "description": "The query to retrieve articles for"
+            }
+          },
+          "required": [
+            "query"
+          ]
+        },
+        "isLibrary": true
       }
     ],
     "prompts": [],
-    "startAgent": "Blog Writer Hub",
+    "startAgent": "Example Agent",
     "mcpServers": [],
     "toolWebhookUrl": ""
   },
-  "lastResponse": null
+  "lastResponse": {
+    "state": {
+      "last_agent_name": "Example Agent",
+      "tokens": {
+        "total": 1581,
+        "prompt": 1533,
+        "completion": 48
+      }
+    },
+    "messages": [
+      {
+        "version": "v1",
+        "chatId": "",
+        "createdAt": "2025-05-08T07:06:55.690Z",
+        "role": "assistant",
+        "tool_calls": [
+          {
+            "id": "call_uzRd4Y1CeBioJ9h26XeGhCvH",
+            "function": {
+              "name": "rag_search",
+              "arguments": "{\"query\":\"article 1\"}"
+            },
+            "type": "function"
+          }
+        ],
+        "agenticSender": "Example Agent",
+        "agenticResponseType": "internal"
+      },
+      {
+        "version": "v1",
+        "chatId": "",
+        "createdAt": "2025-05-08T07:06:55.690Z",
+        "role": "tool",
+        "content": "{\n  \"Information\": [\n    {\n      \"title\": \"text\",\n      \"name\": \"text\",\n      \"content\": \"This is article 1.\",\n      \"docId\": \"681c50f17e7a18621c7215cd\",\n      \"sourceId\": \"681c50f17e7a18621c7215cc\"\n    },\n    {\n      \"title\": \"text\",\n      \"name\": \"text\",\n      \"content\": \"This is article 2.\",\n      \"docId\": \"681c51067e7a18621c7215cf\",\n      \"sourceId\": \"681c51067e7a18621c7215ce\"\n    },\n    {\n      \"title\": \"text\",\n      \"name\": \"text\",\n      \"content\": \"This is article 3.\",\n      \"docId\": \"681c51137e7a18621c7215d1\",\n      \"sourceId\": \"681c51137e7a18621c7215d0\"\n    }\n  ]\n}",
+        "tool_call_id": "call_uzRd4Y1CeBioJ9h26XeGhCvH",
+        "tool_name": ""
+      },
+      {
+        "version": "v1",
+        "chatId": "",
+        "createdAt": "2025-05-08T07:06:57.096Z",
+        "role": "assistant",
+        "content": "Article 1 states: \"This is article 1.\"\n\nIf you need more details or information about another article, please let me know!",
+        "agenticSender": "Example Agent",
+        "agenticResponseType": "external"
+      }
+    ]
+  }
 }

docker-compose.yml

@@ -1,5 +1,13 @@
 version: '3.8'
 
+volumes:
+  uploads:
+    driver: local
+    driver_opts:
+      type: none
+      o: bind
+      device: ./data/uploads
+
 services:
   rowboat:
     build:
@@ -22,9 +30,10 @@ services:
       - COPILOT_API_KEY=${COPILOT_API_KEY}
       - REDIS_URL=redis://redis:6379
       - USE_RAG=${USE_RAG}
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
       - USE_RAG_UPLOADS=${USE_RAG_UPLOADS}
+      - USE_RAG_S3_UPLOADS=${USE_RAG_S3_UPLOADS}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
       - RAG_UPLOADS_S3_BUCKET=${RAG_UPLOADS_S3_BUCKET}
@@ -38,7 +47,10 @@ services:
       - MAX_PROJECTS_PER_USER=${MAX_PROJECTS_PER_USER}
       - VOICE_API_URL=${VOICE_API_URL}
       - PROVIDER_DEFAULT_MODEL=${PROVIDER_DEFAULT_MODEL}
+      - RAG_UPLOADS_DIR=/app/uploads
     restart: unless-stopped
+    volumes:
+      - uploads:/app/uploads
 
   rowboat_agents:
     build:
@@ -51,7 +63,7 @@ services:
       - API_KEY=${AGENTS_API_KEY}
       - REDIS_URL=redis://redis:6379
       - MONGODB_URI=mongodb://mongo:27017/rowboat
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
       - PROVIDER_BASE_URL=${PROVIDER_BASE_URL}
       - PROVIDER_API_KEY=${PROVIDER_API_KEY}
@@ -99,21 +111,28 @@ services:
     build:
       context: ./apps/rowboat
       dockerfile: scripts.Dockerfile
-    command: ["sh", "-c", "npm run setupQdrant && echo 'index created successfully'"]
+    command: ["sh", "-c", "npm run setupQdrant"]
     profiles: [ "setup_qdrant" ]
+    depends_on:
+      qdrant:
+        condition: service_healthy
     environment:
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
+      - EMBEDDING_VECTOR_SIZE=${EMBEDDING_VECTOR_SIZE}
     restart: "no"
 
   delete_qdrant:
     build:
       context: ./apps/rowboat
       dockerfile: scripts.Dockerfile
-    command: ["sh", "-c", "npm run deleteQdrant && echo 'index deleted successfully'"]
+    command: ["sh", "-c", "npm run deleteQdrant"]
     profiles: [ "delete_qdrant" ]
+    depends_on:
+      qdrant:
+        condition: service_healthy
     environment:
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: "no"
 
@@ -125,15 +144,23 @@ services:
     profiles: [ "rag_files_worker" ]
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - EMBEDDING_PROVIDER_BASE_URL=${EMBEDDING_PROVIDER_BASE_URL}
+      - EMBEDDING_PROVIDER_API_KEY=${EMBEDDING_PROVIDER_API_KEY}
+      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
       - MONGODB_CONNECTION_STRING=mongodb://mongo:27017/rowboat
+      - REDIS_URL=redis://redis:6379
       - GOOGLE_API_KEY=${GOOGLE_API_KEY}
       - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
       - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
       - RAG_UPLOADS_S3_BUCKET=${RAG_UPLOADS_S3_BUCKET}
       - RAG_UPLOADS_S3_REGION=${RAG_UPLOADS_S3_REGION}
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
+      - RAG_UPLOADS_DIR=/app/uploads
+      - USE_GEMINI_FILE_PARSING=${USE_GEMINI_FILE_PARSING}
     restart: unless-stopped
+    volumes:
+      - uploads:/app/uploads
 
   rag_urls_worker:
     build:
@@ -143,9 +170,13 @@ services:
     profiles: [ "rag_urls_worker" ]
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - EMBEDDING_PROVIDER_BASE_URL=${EMBEDDING_PROVIDER_BASE_URL}
+      - EMBEDDING_PROVIDER_API_KEY=${EMBEDDING_PROVIDER_API_KEY}
+      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
       - MONGODB_CONNECTION_STRING=mongodb://mongo:27017/rowboat
+      - REDIS_URL=redis://redis:6379
       - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY}
-      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: unless-stopped
 
@@ -157,8 +188,12 @@ services:
     profiles: [ "rag_text_worker" ]
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - EMBEDDING_PROVIDER_BASE_URL=${EMBEDDING_PROVIDER_BASE_URL}
+      - EMBEDDING_PROVIDER_API_KEY=${EMBEDDING_PROVIDER_API_KEY}
+      - EMBEDDING_MODEL=${EMBEDDING_MODEL}
       - MONGODB_CONNECTION_STRING=mongodb://mongo:27017/rowboat
-      - QDRANT_URL=${QDRANT_URL}
+      - REDIS_URL=redis://redis:6379
+      - QDRANT_URL=http://qdrant:6333
       - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: unless-stopped
 
@@ -208,3 +243,21 @@ services:
   #     - ROWBOAT_API_HOST=http://rowboat:3000
   #     - MONGODB_URI=mongodb://mongo:27017/rowboat
   #   restart: unless-stopped
+
+  qdrant:
+    build:
+      context: .
+      dockerfile: Dockerfile.qdrant
+    ports:
+      - "6333:6333"
+    environment:
+      - QDRANT__STORAGE__STORAGE_PATH=/data/qdrant
+    restart: unless-stopped
+    profiles: [ "qdrant" ]
+    volumes:
+      - ./data/qdrant:/data/qdrant
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:6333/healthz"]
+      interval: 5s
+      timeout: 10s
+      retries: 3

start.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# ensure data dirs exist
+mkdir -p data/uploads
+mkdir -p data/qdrant
+mkdir -p data/mongo
+
+# set the following environment variables
+export USE_RAG=true
+export USE_RAG_UPLOADS=true
+
+# Start with the base command and profile flags
+CMD="docker-compose"
+CMD="$CMD --profile setup_qdrant"
+CMD="$CMD --profile qdrant"
+CMD="$CMD --profile rag_text_worker"
+CMD="$CMD --profile rag_files_worker"
+
+# enable rag urls worker
+if [ "$USE_RAG_SCRAPING" = "true" ]; then
+  CMD="$CMD --profile rag_urls_worker"
+fi
+
+# Add more mappings as needed
+# if [ "$SOME_OTHER_ENV" = "true" ]; then
+#   CMD="$CMD --profile some_other_profile"
+# fi
+
+# Add the up and build flags at the end
+CMD="$CMD up --build"
+
+echo "Running: $CMD"
+exec $CMD