diff --git a/apps/copilot/app.py b/apps/copilot/app.py index 40b29321..1f39b55d 100644 --- a/apps/copilot/app.py +++ b/apps/copilot/app.py @@ -1,6 +1,6 @@ from flask import Flask, request, jsonify, Response, stream_with_context from pydantic import BaseModel, ValidationError -from typing import List +from typing import List, Optional from copilot import UserMessage, AssistantMessage, get_response from streaming import get_streaming_response from lib import AgentContext, PromptContext, ToolContext, ChatContext @@ -9,11 +9,20 @@ from functools import wraps from copilot import copilot_instructions_edit_agent import json +class DataSource(BaseModel): + name: str + description: Optional[str] = None + active: bool = True + status: str # 'pending' | 'ready' | 'error' | 'deleted' + error: Optional[str] = None + data: dict # The discriminated union based on type + class ApiRequest(BaseModel): messages: List[UserMessage | AssistantMessage] workflow_schema: str current_workflow_config: str context: AgentContext | PromptContext | ToolContext | ChatContext | None = None + dataSources: Optional[List[DataSource]] = None class ApiResponse(BaseModel): response: str @@ -61,7 +70,8 @@ def chat_stream(): messages=request_data.messages, workflow_schema=request_data.workflow_schema, current_workflow_config=request_data.current_workflow_config, - context=request_data.context + context=request_data.context, + dataSources=request_data.dataSources ) for chunk in stream: diff --git a/apps/copilot/copilot.py b/apps/copilot/copilot.py index 3f4697d0..0b0bb2a1 100644 --- a/apps/copilot/copilot.py +++ b/apps/copilot/copilot.py @@ -1,7 +1,7 @@ from openai import OpenAI from flask import Flask, request, jsonify from pydantic import BaseModel, ValidationError -from typing import List, Dict, Any, Literal +from typing import List, Dict, Any, Literal, Optional import json from lib import AgentContext, PromptContext, ToolContext, ChatContext from client import PROVIDER_COPILOT_MODEL @@ -15,6 +15,14 @@ class AssistantMessage(BaseModel): role: Literal["assistant"] content: str +class DataSource(BaseModel): + name: str + description: Optional[str] = None + active: bool = True + status: str # 'pending' | 'ready' | 'error' | 'deleted' + error: Optional[str] = None + data: dict # The discriminated union based on type + with open('copilot_edit_agent.md', 'r', encoding='utf-8') as file: copilot_instructions_edit_agent = file.read() @@ -23,6 +31,7 @@ def get_response( workflow_schema: str, current_workflow_config: str, context: AgentContext | PromptContext | ToolContext | ChatContext | None = None, + dataSources: Optional[List[DataSource]] = None, copilot_instructions: str = copilot_instructions_edit_agent ) -> str: # if context is provided, create a prompt for the context @@ -53,6 +62,16 @@ def get_response( else: context_prompt = "" + # Add dataSources to the context if provided + data_sources_prompt = "" + if dataSources: + data_sources_prompt = f""" +**NOTE**: The following data sources are available: +```json +{json.dumps([ds.model_dump() for ds in dataSources])} +``` +""" + # add the workflow schema to the system prompt sys_prompt = copilot_instructions.replace("{workflow_schema}", workflow_schema) @@ -66,6 +85,7 @@ The current workflow config is: ``` {context_prompt} +{data_sources_prompt} User: {last_message.content} """ diff --git a/apps/copilot/streaming.py b/apps/copilot/streaming.py index faf97ab0..81342d36 100644 --- a/apps/copilot/streaming.py +++ b/apps/copilot/streaming.py @@ -1,7 +1,7 @@ from openai import OpenAI from flask import Flask, request, jsonify, Response, stream_with_context from pydantic import BaseModel, ValidationError -from typing import List, Dict, Any, Literal +from typing import List, Dict, Any, Literal, Optional import json from lib import AgentContext, PromptContext, ToolContext, ChatContext from client import PROVIDER_COPILOT_MODEL, PROVIDER_DEFAULT_MODEL @@ -15,6 +15,14 @@ class AssistantMessage(BaseModel): role: Literal["assistant"] content: str +class DataSource(BaseModel): + name: str + description: Optional[str] = None + active: bool = True + status: str # 'pending' | 'ready' | 'error' | 'deleted' + error: Optional[str] = None + data: dict # The discriminated union based on type + with open('copilot_multi_agent.md', 'r', encoding='utf-8') as file: copilot_instructions_multi_agent = file.read() @@ -39,6 +47,7 @@ def get_streaming_response( workflow_schema: str, current_workflow_config: str, context: AgentContext | PromptContext | ToolContext | ChatContext | None = None, + dataSources: Optional[List[DataSource]] = None, ) -> Any: # if context is provided, create a prompt for the context if context: @@ -68,6 +77,19 @@ def get_streaming_response( else: context_prompt = "" + # Add dataSources to the context if provided + data_sources_prompt = "" + if dataSources: + print(f"Data sources found at project level: {dataSources}") + data_sources_prompt = f""" +**NOTE**: The following data sources are available: +```json +{json.dumps([ds.model_dump() for ds in dataSources])} +``` +""" + else: + print("No data sources found at project level") + # add the workflow schema to the system prompt sys_prompt = streaming_instructions.replace("{workflow_schema}", workflow_schema) @@ -84,6 +106,7 @@ The current workflow config is: ``` {context_prompt} +{data_sources_prompt} User: {last_message.content} """ @@ -91,7 +114,7 @@ User: {last_message.content} updated_msgs = [{"role": "system", "content": sys_prompt}] + [ message.model_dump() for message in messages ] - + return completions_client.chat.completions.create( model=PROVIDER_COPILOT_MODEL, messages=updated_msgs, diff --git a/apps/rowboat/app/actions/copilot_actions.ts b/apps/rowboat/app/actions/copilot_actions.ts index 47419604..c9717388 100644 --- a/apps/rowboat/app/actions/copilot_actions.ts +++ b/apps/rowboat/app/actions/copilot_actions.ts @@ -2,10 +2,12 @@ import { convertToCopilotWorkflow, convertToCopilotMessage, convertToCopilotApiMessage, convertToCopilotApiChatContext, CopilotAPIResponse, CopilotAPIRequest, - CopilotChatContext, CopilotMessage, CopilotAssistantMessage, CopilotWorkflow + CopilotChatContext, CopilotMessage, CopilotAssistantMessage, CopilotWorkflow, + CopilotDataSource } from "../lib/types/copilot_types"; import { Workflow} from "../lib/types/workflow_types"; +import { DataSource } from "../lib/types/datasource_types"; import { z } from 'zod'; import { zodToJsonSchema } from 'zod-to-json-schema'; import { assert } from "node:console"; @@ -18,7 +20,8 @@ export async function getCopilotResponse( projectId: string, messages: z.infer[], current_workflow_config: z.infer, - context: z.infer | null + context: z.infer | null, + dataSources?: z.infer[] ): Promise<{ message: z.infer; rawRequest: unknown; @@ -35,6 +38,7 @@ export async function getCopilotResponse( workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)), current_workflow_config: JSON.stringify(convertToCopilotWorkflow(current_workflow_config)), context: context ? convertToCopilotApiChatContext(context) : null, + dataSources: dataSources ? dataSources.map(ds => CopilotDataSource.parse(ds)) : undefined, }; console.log(`sending copilot request`, JSON.stringify(request)); @@ -96,7 +100,8 @@ export async function getCopilotResponseStream( projectId: string, messages: z.infer[], current_workflow_config: z.infer, - context: z.infer | null + context: z.infer | null, + dataSources?: z.infer[] ): Promise<{ streamId: string; }> { @@ -111,6 +116,7 @@ export async function getCopilotResponseStream( workflow_schema: JSON.stringify(zodToJsonSchema(CopilotWorkflow)), current_workflow_config: JSON.stringify(convertToCopilotWorkflow(current_workflow_config)), context: context ? convertToCopilotApiChatContext(context) : null, + dataSources: dataSources ? dataSources.map(ds => CopilotDataSource.parse(ds)) : undefined, }; // serialize the request diff --git a/apps/rowboat/app/actions/datasource_actions.ts b/apps/rowboat/app/actions/datasource_actions.ts index bfe5feed..cc6fe18b 100644 --- a/apps/rowboat/app/actions/datasource_actions.ts +++ b/apps/rowboat/app/actions/datasource_actions.ts @@ -43,11 +43,13 @@ export async function listDataSources(projectId: string): Promise['data'], status?: 'pending' | 'ready', }): Promise>> { @@ -57,6 +59,7 @@ export async function createDataSource({ projectId: projectId, active: true, name: name, + description, createdAt: (new Date()).toISOString(), attempts: 0, status: status, @@ -354,3 +357,28 @@ export async function getUploadUrlsForFilesDataSource( return urls; } + +export async function updateDataSource({ + projectId, + sourceId, + description, +}: { + projectId: string, + sourceId: string, + description: string, +}) { + await projectAuthCheck(projectId); + await getDataSource(projectId, sourceId); + + await dataSourcesCollection.updateOne({ + _id: new ObjectId(sourceId), + }, { + $set: { + description, + lastUpdatedAt: (new Date()).toISOString(), + }, + $inc: { + version: 1, + }, + }); +} diff --git a/apps/rowboat/app/lib/types/copilot_types.ts b/apps/rowboat/app/lib/types/copilot_types.ts index 457eb1c4..21245205 100644 --- a/apps/rowboat/app/lib/types/copilot_types.ts +++ b/apps/rowboat/app/lib/types/copilot_types.ts @@ -3,11 +3,23 @@ import { Workflow } from "./workflow_types"; import { apiV1 } from "rowboat-shared" import { AgenticAPIChatMessage } from "./agents_api_types"; import { convertToAgenticAPIChatMessages } from "./agents_api_types"; +import { DataSource } from "./datasource_types"; + +// Create a filtered version of DataSource for copilot +export const CopilotDataSource = DataSource.omit({ + projectId: true, + version: true, + attempts: true, + createdAt: true, + lastUpdatedAt: true, + pendingRefresh: true, +}); export const CopilotWorkflow = Workflow.omit({ lastUpdatedAt: true, projectId: true, -});export const CopilotUserMessage = z.object({ +}); +export const CopilotUserMessage = z.object({ role: z.literal('user'), content: z.string(), }); @@ -77,6 +89,7 @@ export const CopilotAPIRequest = z.object({ workflow_schema: z.string(), current_workflow_config: z.string(), context: CopilotApiChatContext.nullable(), + dataSources: z.array(CopilotDataSource).optional(), }); export const CopilotAPIResponse = z.union([ z.object({ diff --git a/apps/rowboat/app/lib/types/datasource_types.ts b/apps/rowboat/app/lib/types/datasource_types.ts index 5e4ce7a2..70cc729c 100644 --- a/apps/rowboat/app/lib/types/datasource_types.ts +++ b/apps/rowboat/app/lib/types/datasource_types.ts @@ -2,6 +2,7 @@ import { z } from "zod"; export const DataSource = z.object({ name: z.string(), + description: z.string().optional(), projectId: z.string(), active: z.boolean().default(true), status: z.union([ diff --git a/apps/rowboat/app/projects/[projectId]/copilot/app.tsx b/apps/rowboat/app/projects/[projectId]/copilot/app.tsx index 25826170..251dba04 100644 --- a/apps/rowboat/app/projects/[projectId]/copilot/app.tsx +++ b/apps/rowboat/app/projects/[projectId]/copilot/app.tsx @@ -5,6 +5,7 @@ import { useRef, useState, createContext, useContext, useCallback, forwardRef, u import { CopilotChatContext } from "../../../lib/types/copilot_types"; import { CopilotMessage } from "../../../lib/types/copilot_types"; import { Workflow } from "@/app/lib/types/workflow_types"; +import { DataSource } from "@/app/lib/types/datasource_types"; import { z } from "zod"; import { Action as WorkflowDispatch } from "../workflow/workflow_editor"; import { Panel } from "@/components/common/panel-common"; @@ -30,6 +31,7 @@ interface AppProps { onCopyJson?: (data: { messages: any[] }) => void; onMessagesChange?: (messages: z.infer[]) => void; isInitialState?: boolean; + dataSources?: z.infer[]; } const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({ @@ -40,6 +42,7 @@ const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({ onCopyJson, onMessagesChange, isInitialState = false, + dataSources, }, ref) { const [messages, setMessages] = useState[]>([]); const [discardContext, setDiscardContext] = useState(false); @@ -63,7 +66,8 @@ const App = forwardRef<{ handleCopyChat: () => void }, AppProps>(function App({ } = useCopilot({ projectId, workflow: workflowRef.current, - context: effectiveContext + context: effectiveContext, + dataSources: dataSources }); // Store latest start/cancel functions in refs @@ -196,12 +200,14 @@ export function Copilot({ chatContext = undefined, dispatch, isInitialState = false, + dataSources, }: { projectId: string; workflow: z.infer; chatContext?: z.infer; dispatch: (action: WorkflowDispatch) => void; isInitialState?: boolean; + dataSources?: z.infer[]; }) { const [copilotKey, setCopilotKey] = useState(0); const [showCopySuccess, setShowCopySuccess] = useState(false); @@ -277,6 +283,7 @@ export function Copilot({ onCopyJson={handleCopyJson} onMessagesChange={setMessages} isInitialState={isInitialState} + dataSources={dataSources} /> diff --git a/apps/rowboat/app/projects/[projectId]/copilot/use-copilot.tsx b/apps/rowboat/app/projects/[projectId]/copilot/use-copilot.tsx index b36fcb27..f1d4e262 100644 --- a/apps/rowboat/app/projects/[projectId]/copilot/use-copilot.tsx +++ b/apps/rowboat/app/projects/[projectId]/copilot/use-copilot.tsx @@ -2,12 +2,14 @@ import { useCallback, useRef, useState } from "react"; import { getCopilotResponseStream } from "@/app/actions/copilot_actions"; import { CopilotMessage } from "@/app/lib/types/copilot_types"; import { Workflow } from "@/app/lib/types/workflow_types"; +import { DataSource } from "@/app/lib/types/datasource_types"; import { z } from "zod"; interface UseCopilotParams { projectId: string; workflow: z.infer; context: any; + dataSources?: z.infer[]; } interface UseCopilotResult { @@ -21,7 +23,7 @@ interface UseCopilotResult { cancel: () => void; } -export function useCopilot({ projectId, workflow, context }: UseCopilotParams): UseCopilotResult { +export function useCopilot({ projectId, workflow, context, dataSources }: UseCopilotParams): UseCopilotResult { const [streamingResponse, setStreamingResponse] = useState(''); const [loading, setLoading] = useState(false); const [error, setError] = useState(null); @@ -41,7 +43,7 @@ export function useCopilot({ projectId, workflow, context }: UseCopilotParams): setLoading(true); try { - const res = await getCopilotResponseStream(projectId, messages, workflow, context || null); + const res = await getCopilotResponseStream(projectId, messages, workflow, context || null, dataSources); const eventSource = new EventSource(`/api/copilot-stream-response/${res.streamId}`); eventSource.onmessage = (event) => { @@ -71,7 +73,7 @@ export function useCopilot({ projectId, workflow, context }: UseCopilotParams): setError('Failed to initiate stream'); setLoading(false); } - }, [projectId, workflow, context]); + }, [projectId, workflow, context, dataSources]); const cancel = useCallback(() => { cancelRef.current?.(); diff --git a/apps/rowboat/app/projects/[projectId]/sources/[sourceId]/source-page.tsx b/apps/rowboat/app/projects/[projectId]/sources/[sourceId]/source-page.tsx index c9073346..ce087cf2 100644 --- a/apps/rowboat/app/projects/[projectId]/sources/[sourceId]/source-page.tsx +++ b/apps/rowboat/app/projects/[projectId]/sources/[sourceId]/source-page.tsx @@ -10,12 +10,14 @@ import { DataSourceIcon } from "../../../../lib/components/datasource-icon"; import { z } from "zod"; import { ScrapeSource } from "../components/scrape-source"; import { FilesSource } from "../components/files-source"; -import { getDataSource } from "../../../../actions/datasource_actions"; +import { getDataSource, updateDataSource } from "../../../../actions/datasource_actions"; import { TextSource } from "../components/text-source"; import { Panel } from "@/components/common/panel-common"; import { Section, SectionRow, SectionLabel, SectionContent } from "../components/section"; import Link from "next/link"; import { BackIcon } from "../../../../lib/components/icons"; +import { Textarea } from "@/components/ui/textarea"; +import { CheckIcon } from "lucide-react"; export function SourcePage({ sourceId, @@ -26,6 +28,7 @@ export function SourcePage({ }) { const [source, setSource] = useState> | null>(null); const [isLoading, setIsLoading] = useState(true); + const [showSaveSuccess, setShowSaveSuccess] = useState(false); async function handleReload() { setIsLoading(true); @@ -122,6 +125,57 @@ export function SourcePage({ + + Name + +
+ {source.name} +
+ + + + + Description + +
{ + const description = formData.get('description') as string; + await updateDataSource({ + projectId, + sourceId, + description, + }); + handleReload(); + setShowSaveSuccess(true); + setTimeout(() => setShowSaveSuccess(false), 2000); + }} + className="w-full" + > +