From aa95b590e24996e9097f64e2ddc0a0aac4f97f8c Mon Sep 17 00:00:00 2001 From: arkml Date: Tue, 29 Jul 2025 14:29:22 +0530 Subject: [PATCH] move data-source ops to a modal inside build view (#186) * added data source modal to build view * fixed scrape url and upload files showing up in the data modal in the build view * autoupdate data source status * keep focus in build view on deleting a data source * removed the rag tab and allow multiple file uploads * show scrapred urls and uploaded files * show text in the text data source * fixed status refreshes when using the detail view of data source * fix status refresh in the data panel * cleanup * fixed review comments on backend changes * moved the input text field in text data source to the new ui element * Fix PR review: Replace router.push with data sources modal in agent config - Add onOpenDataSourcesModal prop to AgentConfig component - Update 'Go to Data Sources' button to open modal instead of navigating - Change button text to 'Add Data Source' for clarity - Use forwardRef and useImperativeHandle to expose modal control - Remove router dependency from agent_config.tsx Addresses reviewer comment about using modal instead of route change. * fixed build issue * fixed nit comments * fixed line break --- .../rowboat/app/actions/datasource_actions.ts | 3 - apps/rowboat/app/billing/layout.tsx | 2 +- apps/rowboat/app/onboarding/layout.tsx | 2 +- .../[projectId]/entities/agent_config.tsx | 8 +- .../entities/datasource_config.tsx | 922 ++++++++++++++++++ .../projects/[projectId]/sources/new/form.tsx | 52 +- .../app/projects/[projectId]/workflow/app.tsx | 37 + .../workflow/components/DataSourcesModal.tsx | 117 +++ .../[projectId]/workflow/entity_list.tsx | 226 ++++- .../projects/[projectId]/workflow/page.tsx | 5 +- .../[projectId]/workflow/workflow_editor.tsx | 49 +- apps/rowboat/app/projects/layout.tsx | 4 +- .../projects/layout/components/app-layout.tsx | 4 +- .../projects/layout/components/sidebar.tsx | 10 +- apps/rowboat/app/projects/layout/menu.tsx | 13 +- apps/rowboat/app/projects/layout/nav.tsx | 4 +- 16 files changed, 1387 insertions(+), 71 deletions(-) create mode 100644 apps/rowboat/app/projects/[projectId]/entities/datasource_config.tsx create mode 100644 apps/rowboat/app/projects/[projectId]/workflow/components/DataSourcesModal.tsx diff --git a/apps/rowboat/app/actions/datasource_actions.ts b/apps/rowboat/app/actions/datasource_actions.ts index 4e59292d..daab970d 100644 --- a/apps/rowboat/app/actions/datasource_actions.ts +++ b/apps/rowboat/app/actions/datasource_actions.ts @@ -1,5 +1,4 @@ 'use server'; -import { redirect } from "next/navigation"; import { ObjectId, WithId } from "mongodb"; import { dataSourcesCollection, dataSourceDocsCollection } from "../lib/mongodb"; import { z } from 'zod'; @@ -132,8 +131,6 @@ export async function deleteDataSource(projectId: string, sourceId: string) { version: 1, }, }); - - redirect(`/projects/${projectId}/sources`); } export async function toggleDataSource(projectId: string, sourceId: string, active: boolean) { diff --git a/apps/rowboat/app/billing/layout.tsx b/apps/rowboat/app/billing/layout.tsx index 3547f9ea..610e6d04 100644 --- a/apps/rowboat/app/billing/layout.tsx +++ b/apps/rowboat/app/billing/layout.tsx @@ -6,7 +6,7 @@ export default function Layout({ children: React.ReactNode; }>) { return ( - + {children} ); diff --git a/apps/rowboat/app/onboarding/layout.tsx b/apps/rowboat/app/onboarding/layout.tsx index 3547f9ea..610e6d04 100644 --- a/apps/rowboat/app/onboarding/layout.tsx +++ b/apps/rowboat/app/onboarding/layout.tsx @@ -6,7 +6,7 @@ export default function Layout({ children: React.ReactNode; }>) { return ( - + {children} ); diff --git a/apps/rowboat/app/projects/[projectId]/entities/agent_config.tsx b/apps/rowboat/app/projects/[projectId]/entities/agent_config.tsx index 65d26b68..4147b433 100644 --- a/apps/rowboat/app/projects/[projectId]/entities/agent_config.tsx +++ b/apps/rowboat/app/projects/[projectId]/entities/agent_config.tsx @@ -23,7 +23,6 @@ import { Info } from "lucide-react"; import { useCopilot } from "../copilot/use-copilot"; import { BillingUpgradeModal } from "@/components/common/billing-upgrade-modal"; import { ModelsResponse } from "@/app/lib/types/billing_types"; -import { useRouter } from "next/navigation"; import { SectionCard } from "@/components/common/section-card"; // Common section header styles @@ -49,6 +48,7 @@ export function AgentConfig({ useRag, triggerCopilotChat, eligibleModels, + onOpenDataSourcesModal, }: { projectId: string, workflow: z.infer, @@ -63,6 +63,7 @@ export function AgentConfig({ useRag: boolean, triggerCopilotChat: (message: string) => void, eligibleModels: z.infer | "*", + onOpenDataSourcesModal?: () => void, }) { const [isAdvancedConfigOpen, setIsAdvancedConfigOpen] = useState(false); const [showGenerateModal, setShowGenerateModal] = useState(false); @@ -75,7 +76,6 @@ export function AgentConfig({ const [showRagCta, setShowRagCta] = useState(false); const [previousRagSources, setPreviousRagSources] = useState([]); const [billingError, setBillingError] = useState(null); - const router = useRouter(); const [showSavedBanner, setShowSavedBanner] = useState(false); const { @@ -706,11 +706,11 @@ export function AgentConfig({ onClick={(e) => { e.preventDefault(); e.stopPropagation(); - router.push(`/projects/${projectId}/sources`); + onOpenDataSourcesModal?.(); }} startContent={} > - Go to RAG Sources + Add Data Source diff --git a/apps/rowboat/app/projects/[projectId]/entities/datasource_config.tsx b/apps/rowboat/app/projects/[projectId]/entities/datasource_config.tsx new file mode 100644 index 00000000..dd8ac188 --- /dev/null +++ b/apps/rowboat/app/projects/[projectId]/entities/datasource_config.tsx @@ -0,0 +1,922 @@ +"use client"; +import { WithStringId } from "../../../lib/types/types"; +import { DataSource } from "../../../lib/types/datasource_types"; +import { z } from "zod"; +import { XIcon, FileIcon, GlobeIcon, AlertTriangle, CheckCircle, Circle, ExternalLinkIcon, Type, PlusIcon, Edit3Icon, DownloadIcon, Trash2 } from "lucide-react"; +import { useState, useEffect, useCallback } from "react"; +import { Panel } from "@/components/common/panel-common"; +import { Button } from "@/components/ui/button"; +import { DataSourceIcon } from "@/app/lib/components/datasource-icon"; +import { Tooltip } from "@heroui/react"; +import { getDataSource, listDocsInDataSource, deleteDocsFromDataSource, getDownloadUrlForFile, addDocsToDataSource, getUploadUrlsForFilesDataSource } from "@/app/actions/datasource_actions"; +import { InputField } from "@/app/lib/components/input-field"; +import { DataSourceDoc } from "../../../lib/types/datasource_types"; +import { RelativeTime } from "@primer/react"; +import { Pagination, Spinner, Button as HeroButton, Textarea as HeroTextarea } from "@heroui/react"; +import { useDropzone } from "react-dropzone"; + +export function DataSourceConfig({ + dataSourceId, + handleClose, + onDataSourceUpdate +}: { + dataSourceId: string, + handleClose: () => void, + onDataSourceUpdate?: () => void +}) { + const [dataSource, setDataSource] = useState> | null>(null); + const [loading, setLoading] = useState(true); + const [error, setError] = useState(null); + + // Files-related state + const [files, setFiles] = useState>[]>([]); + const [filesLoading, setFilesLoading] = useState(false); + const [filesPage, setFilesPage] = useState(1); + const [filesTotal, setFilesTotal] = useState(0); + const [projectId, setProjectId] = useState(''); + + useEffect(() => { + async function loadDataSource() { + try { + setLoading(true); + // Extract projectId from the current URL + const pathParts = window.location.pathname.split('/'); + const currentProjectId = pathParts[2]; // /projects/[projectId]/workflow + setProjectId(currentProjectId); + + const ds = await getDataSource(currentProjectId, dataSourceId); + setDataSource(ds); + + // Load files if it's a files data source + if (ds.data.type === 'files_local' || ds.data.type === 'files_s3') { + await loadFiles(currentProjectId, dataSourceId, 1); + } + + // Load URLs if it's a URLs data source + if (ds.data.type === 'urls') { + await loadUrls(currentProjectId, dataSourceId, 1); + } + + // Load text content if it's a text data source + if (ds.data.type === 'text') { + await loadTextContent(currentProjectId, dataSourceId); + } + } catch (err) { + console.error('Failed to load data source:', err); + setError('Failed to load data source details'); + } finally { + setLoading(false); + } + } + + loadDataSource(); + }, [dataSourceId]); + + // Auto-refresh data source status when it's pending + useEffect(() => { + let ignore = false; + let timeout: NodeJS.Timeout | null = null; + + if (!dataSource || !projectId) { + return; + } + + if (dataSource.status !== 'pending') { + return; + } + + async function refreshStatus() { + if (timeout) { + clearTimeout(timeout); + } + + try { + const updatedSource = await getDataSource(projectId, dataSourceId); + if (!ignore) { + setDataSource(updatedSource); + onDataSourceUpdate?.(); // Notify parent of status change + + // Continue polling if still pending + if (updatedSource.status === 'pending') { + timeout = setTimeout(refreshStatus, 5000); // Poll every 5 seconds + } + } + } catch (err) { + console.error('Failed to refresh data source status:', err); + // Retry after a longer delay on error + if (!ignore) { + timeout = setTimeout(refreshStatus, 10000); + } + } + } + + // Start polling after a short delay + timeout = setTimeout(refreshStatus, 5000); + + return () => { + ignore = true; + if (timeout) { + clearTimeout(timeout); + } + }; + }, [dataSource, projectId, dataSourceId, onDataSourceUpdate]); + + // Helper function to update data source and notify parent + const updateDataSourceAndNotify = useCallback(async () => { + try { + const updatedSource = await getDataSource(projectId, dataSourceId); + setDataSource(updatedSource); + onDataSourceUpdate?.(); + } catch (err) { + console.error('Failed to reload data source:', err); + } + }, [projectId, dataSourceId, onDataSourceUpdate]); + + // Load files function + const loadFiles = async (projectId: string, sourceId: string, page: number) => { + try { + setFilesLoading(true); + const { files, total } = await listDocsInDataSource({ + projectId, + sourceId, + page, + limit: 10, + }); + setFiles(files); + setFilesTotal(total); + setFilesPage(page); + } catch (err) { + console.error('Failed to load files:', err); + } finally { + setFilesLoading(false); + } + }; + + // URLs-related state + const [urls, setUrls] = useState>[]>([]); + const [urlsLoading, setUrlsLoading] = useState(false); + const [urlsPage, setUrlsPage] = useState(1); + const [urlsTotal, setUrlsTotal] = useState(0); + + // Text-related state + const [textContent, setTextContent] = useState(''); + const [textLoading, setTextLoading] = useState(false); + const [savingText, setSavingText] = useState(false); + + // URL form state + const [showAddUrlForm, setShowAddUrlForm] = useState(false); + const [addingUrls, setAddingUrls] = useState(false); + + // File upload state + const [uploadingFiles, setUploadingFiles] = useState(false); + + // Load URLs function + const loadUrls = async (projectId: string, sourceId: string, page: number) => { + try { + setUrlsLoading(true); + const { files, total } = await listDocsInDataSource({ + projectId, + sourceId, + page, + limit: 10, + }); + setUrls(files); + setUrlsTotal(total); + setUrlsPage(page); + } catch (err) { + console.error('Failed to load URLs:', err); + } finally { + setUrlsLoading(false); + } + }; + + // Load text content function + const loadTextContent = async (projectId: string, sourceId: string) => { + try { + setTextLoading(true); + const { files } = await listDocsInDataSource({ + projectId, + sourceId, + limit: 1, + }); + + if (files.length > 0 && files[0].data.type === 'text') { + setTextContent(files[0].data.content); + } else { + setTextContent(''); + } + } catch (err) { + console.error('Failed to load text content:', err); + setTextContent(''); + } finally { + setTextLoading(false); + } + }; + + // Handle file deletion + const handleDeleteFile = async (fileId: string) => { + if (!window.confirm('Are you sure you want to delete this file?')) return; + + try { + await deleteDocsFromDataSource({ + projectId, + sourceId: dataSourceId, + docIds: [fileId], + }); + // Reload files + await loadFiles(projectId, dataSourceId, filesPage); + + // Reload data source to get updated status + await updateDataSourceAndNotify(); + } catch (err) { + console.error('Failed to delete file:', err); + } + }; + + // Handle file download + const handleDownloadFile = async (fileId: string) => { + try { + const url = await getDownloadUrlForFile(projectId, dataSourceId, fileId); + window.open(url, '_blank'); + } catch (err) { + console.error('Failed to download file:', err); + } + }; + + // Handle page change + const handlePageChange = (page: number) => { + loadFiles(projectId, dataSourceId, page); + }; + + // Handle URL deletion + const handleDeleteUrl = async (urlId: string) => { + if (!window.confirm('Are you sure you want to delete this URL?')) return; + + try { + await deleteDocsFromDataSource({ + projectId, + sourceId: dataSourceId, + docIds: [urlId], + }); + // Reload URLs + await loadUrls(projectId, dataSourceId, urlsPage); + + // Reload data source to get updated status + await updateDataSourceAndNotify(); + } catch (err) { + console.error('Failed to delete URL:', err); + } + }; + + // Handle URL page change + const handleUrlPageChange = (page: number) => { + loadUrls(projectId, dataSourceId, page); + }; + + // Handle text content update + const handleUpdateTextContent = async (newContent: string) => { + setSavingText(true); + try { + // Delete existing text doc if it exists + const { files } = await listDocsInDataSource({ + projectId, + sourceId: dataSourceId, + limit: 1, + }); + + if (files.length > 0) { + await deleteDocsFromDataSource({ + projectId, + sourceId: dataSourceId, + docIds: [files[0]._id], + }); + } + + // Add new text doc + await addDocsToDataSource({ + projectId, + sourceId: dataSourceId, + docData: [{ + name: 'text', + data: { + type: 'text', + content: newContent, + }, + }], + }); + + setTextContent(newContent); + + // Reload data source to get updated status + await updateDataSourceAndNotify(); + } catch (err) { + console.error('Failed to save text:', err); + } finally { + setSavingText(false); + } + }; + + // Handle URL addition + const handleAddUrls = async (formData: FormData) => { + setAddingUrls(true); + try { + const urls = formData.get('urls') as string; + const urlsArray = urls.split('\n') + .map(url => url.trim()) + .filter(url => url.length > 0); + const first100Urls = urlsArray.slice(0, 100); + + await addDocsToDataSource({ + projectId, + sourceId: dataSourceId, + docData: first100Urls.map(url => ({ + name: url, + data: { + type: 'url', + url, + }, + })), + }); + + setShowAddUrlForm(false); + await loadUrls(projectId, dataSourceId, urlsPage); + + // Reload data source to get updated status + await updateDataSourceAndNotify(); + } catch (err) { + console.error('Failed to add URLs:', err); + } finally { + setAddingUrls(false); + } + }; + + // Handle file upload + const onFileDrop = useCallback(async (acceptedFiles: File[]) => { + if (!dataSource) return; + + setUploadingFiles(true); + try { + const urls = await getUploadUrlsForFilesDataSource(projectId, dataSourceId, acceptedFiles.map(file => ({ + name: file.name, + type: file.type, + size: file.size, + }))); + + // Upload files in parallel + await Promise.all(acceptedFiles.map(async (file, index) => { + await fetch(urls[index].uploadUrl, { + method: 'PUT', + body: file, + headers: { + 'Content-Type': file.type, + }, + }); + })); + + // After successful uploads, update the database with file information + let docData: { + _id: string, + name: string, + data: z.infer['data'] + }[] = []; + + const isS3 = dataSource.data.type === 'files_s3'; + + if (isS3) { + docData = acceptedFiles.map((file, index) => ({ + _id: urls[index].fileId, + name: file.name, + data: { + type: 'file_s3' as const, + name: file.name, + size: file.size, + mimeType: file.type, + s3Key: urls[index].path, + }, + })); + } else { + docData = acceptedFiles.map((file, index) => ({ + _id: urls[index].fileId, + name: file.name, + data: { + type: 'file_local' as const, + name: file.name, + size: file.size, + mimeType: file.type, + }, + })); + } + + await addDocsToDataSource({ + projectId, + sourceId: dataSourceId, + docData, + }); + + await loadFiles(projectId, dataSourceId, filesPage); + + // Reload data source to get updated status + await updateDataSourceAndNotify(); + } catch (error) { + console.error('Upload failed:', error); + } finally { + setUploadingFiles(false); + } + }, [projectId, dataSourceId, dataSource, filesPage, updateDataSourceAndNotify]); + + const { getRootProps, getInputProps, isDragActive } = useDropzone({ + onDrop: onFileDrop, + disabled: uploadingFiles, + accept: { + 'application/pdf': ['.pdf'], + }, + }); + + if (loading) { + return ( + +
+ Loading Data Source... +
+ + + } + > +
+
+
+ + ); + } + + if (error || !dataSource) { + return ( + +
+ Error Loading Data Source +
+ + + } + > +
+
+ +

{error || 'Data source not found'}

+
+
+ + ); + } + + // Determine status + const isActive = dataSource.active && dataSource.status === 'ready'; + const isPending = dataSource.status === 'pending'; + const isError = dataSource.status === 'error'; + + // Status indicator + const statusIndicator = () => { + if (isPending) { + return ( +
+ + Processing +
+ ); + } else if (isError) { + return ( +
+ + Error +
+ ); + } else if (isActive) { + return ( +
+ + Active +
+ ); + } else { + return ( +
+ + Inactive +
+ ); + } + }; + + // Type display name + const getTypeDisplayName = (type: string) => { + switch (type) { + case 'urls': return 'Scraped URLs'; + case 'files_local': return 'Local Files'; + case 'files_s3': return 'S3 Files'; + case 'text': return 'Text Content'; + default: return type; + } + }; + + return ( + +
+ +
+
+ {dataSource.name} +
+
+ Data Source +
+
+
+ + + } + > +
+
+ {/* Status Section */} +
+

Status

+ {statusIndicator()} + {isError && dataSource.error && ( +
+

{dataSource.error}

+
+ )} +
+ + {/* Basic Information */} +
+

Information

+
+
+ Type: + {getTypeDisplayName(dataSource.data.type)} +
+
+ Status: +
+ {statusIndicator()} +
+
+
+ Created: + + {new Date(dataSource.createdAt).toLocaleDateString()} + +
+ {dataSource.lastUpdatedAt && ( +
+ Last Updated: + + {new Date(dataSource.lastUpdatedAt).toLocaleDateString()} + +
+ )} +
+
+ + {/* Description */} + {dataSource.description && ( +
+

Description

+

+ {dataSource.description} +

+
+ )} + + {/* Files Section (for file-type data sources) */} + {(dataSource.data.type === 'files_local' || dataSource.data.type === 'files_s3') && ( +
+
+

+ Files ({filesTotal}) +

+
+ + {/* File Upload Area */} +
+ + {uploadingFiles ? ( +
+ +

Uploading files...

+
+ ) : isDragActive ? ( +

Drop the files here...

+ ) : ( +
+
+ +

Drag and drop files here, or click to select files

+
+

+ Only PDF files are supported for now. +

+
+ )} +
+ + {filesLoading ? ( +
+ +

Loading files...

+
+ ) : files.length === 0 ? ( +
+ +

No files uploaded yet

+
+ ) : ( +
+ {files.map((file) => ( +
+
+ +
+

+ {file.name} +

+

+ + {file.data.type === 'file_local' && ' • Local'} + {file.data.type === 'file_s3' && ' • S3'} +

+
+
+
+ {(file.data.type === 'file_local' || file.data.type === 'file_s3') && ( + + + + )} + + + +
+
+ ))} + + {/* Pagination */} + {filesTotal > 10 && ( +
+ +
+ )} +
+ )} +
+ )} + + {/* URLs Section (for URL-type data sources) */} + {dataSource.data.type === 'urls' && ( +
+
+

+ URLs ({urlsTotal}) +

+
+ + {/* Add URLs Button/Form */} + {!showAddUrlForm ? ( + setShowAddUrlForm(true)} + variant="bordered" + size="sm" + startContent={} + > + Add URLs + + ) : ( +
+
+ + +
+
+ : undefined} + > + {addingUrls ? 'Adding...' : 'Add URLs'} + + setShowAddUrlForm(false)} + isDisabled={addingUrls} + > + Cancel + +
+
+ )} + + {urlsLoading ? ( +
+ +

Loading URLs...

+
+ ) : urls.length === 0 ? ( +
+ +

No URLs added yet

+
+ ) : ( +
+ {urls.map((url) => ( +
+
+ +
+
+

+ {url.name} +

+ {url.data.type === 'url' && ( + + + + )} +
+

+ +

+
+
+
+ + + +
+
+ ))} + + {/* Pagination */} + {urlsTotal > 10 && ( +
+ +
+ )} +
+ )} +
+ )} + + {/* Text Content Section (for text-type data sources) */} + {dataSource.data.type === 'text' && ( +
+
+

+ Text Content +

+
+ + {textLoading ? ( +
+ +

Loading content...

+
+ ) : ( +
+ + + {savingText && ( +
+ Saving... +
+ )} +
+ )} +
+ )} + + {/* Usage Information */} +
+

Usage

+
+
+
+ + + +
+
+

Using this data source

+

To use this data source in your agents, go to the RAG tab in individual agent settings and connect this data source.

+
+
+
+
+
+
+ + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx index 3f222299..3ef012f4 100644 --- a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx +++ b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx @@ -6,7 +6,6 @@ import { createDataSource, addDocsToDataSource } from "../../../../actions/datas import { FormStatusButton } from "../../../../lib/components/form-status-button"; import { DataSourceIcon } from "../../../../lib/components/datasource-icon"; import { PlusIcon } from "lucide-react"; -import { useRouter } from "next/navigation"; import { Dropdown } from "@/components/ui/dropdown"; import { Panel } from "@/components/common/panel-common"; @@ -15,14 +14,17 @@ export function Form({ useRagUploads, useRagS3Uploads, useRagScraping, + onSuccess, + hidePanel = false, }: { projectId: string; useRagUploads: boolean; useRagS3Uploads: boolean; useRagScraping: boolean; + onSuccess?: (sourceId: string) => void; + hidePanel?: boolean; }) { const [sourceType, setSourceType] = useState(""); - const router = useRouter(); let dropdownOptions = [ { @@ -79,7 +81,9 @@ export function Form({ }, })), }); - router.push(`/projects/${projectId}/sources/${source._id}`); + if (onSuccess) { + onSuccess(source._id); + } } async function createFilesDataSource(formData: FormData) { @@ -92,7 +96,9 @@ export function Form({ }, }); - router.push(`/projects/${projectId}/sources/${source._id}`); + if (onSuccess) { + onSuccess(source._id); + } } async function createTextDataSource(formData: FormData) { @@ -119,21 +125,14 @@ export function Form({ }], }); - router.push(`/projects/${projectId}/sources/${source._id}`); + if (onSuccess) { + onSuccess(source._id); + } } - return ( - -
- NEW DATA SOURCE -
- - } - > -
-
+ const formContent = ( +
+
} -
+
+ ); + + if (hidePanel) { + return formContent; + } + + return ( + +
+ NEW DATA SOURCE +
+
+ } + > + {formContent} ); } \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/workflow/app.tsx b/apps/rowboat/app/projects/[projectId]/workflow/app.tsx index ba0e0df9..c3933479 100644 --- a/apps/rowboat/app/projects/[projectId]/workflow/app.tsx +++ b/apps/rowboat/app/projects/[projectId]/workflow/app.tsx @@ -16,10 +16,16 @@ import { ModelsResponse } from "@/app/lib/types/billing_types"; export function App({ projectId, useRag, + useRagUploads, + useRagS3Uploads, + useRagScraping, defaultModel, }: { projectId: string; useRag: boolean; + useRagUploads: boolean; + useRagS3Uploads: boolean; + useRagScraping: boolean; defaultModel: string; }) { const [mode, setMode] = useState<'draft' | 'live'>('draft'); @@ -71,6 +77,33 @@ export function App({ setProjectMcpServers(projectConfig.mcpServers); } }, [projectId]); + + const handleDataSourcesUpdate = useCallback(async () => { + // Refresh data sources + const updatedDataSources = await listDataSources(projectId); + setDataSources(updatedDataSources); + }, [projectId]); + + // Auto-update data sources when there are pending ones + useEffect(() => { + if (!dataSources) return; + + const hasPendingSources = dataSources.some(ds => ds.status === 'pending'); + if (!hasPendingSources) return; + + const interval = setInterval(async () => { + const updatedDataSources = await listDataSources(projectId); + setDataSources(updatedDataSources); + + // Stop polling if no more pending sources + const stillHasPending = updatedDataSources.some(ds => ds.status === 'pending'); + if (!stillHasPending) { + clearInterval(interval); + } + }, 7000); // Poll every 7 seconds (reduced from 3) + + return () => clearInterval(interval); + }, [dataSources, projectId]); // Add this useEffect for initial load useEffect(() => { loadData(); @@ -101,12 +134,16 @@ export function App({ dataSources={dataSources} projectConfig={projectConfig || project} useRag={useRag} + useRagUploads={useRagUploads} + useRagS3Uploads={useRagS3Uploads} + useRagScraping={useRagScraping} mcpServerUrls={projectMcpServers} defaultModel={defaultModel} eligibleModels={eligibleModels} onChangeMode={handleSetMode} onRevertToLive={handleRevertToLive} onProjectToolsUpdated={handleProjectToolsUpdate} + onDataSourcesUpdated={handleDataSourcesUpdate} />} } diff --git a/apps/rowboat/app/projects/[projectId]/workflow/components/DataSourcesModal.tsx b/apps/rowboat/app/projects/[projectId]/workflow/components/DataSourcesModal.tsx new file mode 100644 index 00000000..588e4750 --- /dev/null +++ b/apps/rowboat/app/projects/[projectId]/workflow/components/DataSourcesModal.tsx @@ -0,0 +1,117 @@ +'use client'; + +import React, { useState, useEffect } from 'react'; +import { Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from '@heroui/react'; +import { Button } from '@/components/ui/button'; +import { Form } from '../../sources/new/form'; +import { FilesSource } from '../../sources/components/files-source'; +import { getDataSource } from '../../../../actions/datasource_actions'; +import { WithStringId } from '../../../../lib/types/types'; +import { DataSource } from '../../../../lib/types/datasource_types'; +import { z } from 'zod'; + +interface DataSourcesModalProps { + isOpen: boolean; + onClose: () => void; + projectId: string; + onDataSourceAdded?: () => void; + useRagUploads: boolean; + useRagS3Uploads: boolean; + useRagScraping: boolean; +} + +export function DataSourcesModal({ + isOpen, + onClose, + projectId, + onDataSourceAdded, + useRagUploads, + useRagS3Uploads, + useRagScraping +}: DataSourcesModalProps) { + const [currentView, setCurrentView] = useState<'form' | 'upload'>('form'); + const [createdSource, setCreatedSource] = useState> | null>(null); + + const handleDataSourceCreated = async (sourceId: string) => { + // Get the created data source + const source = await getDataSource(projectId, sourceId); + + // If it's a files data source, show the upload interface + if (source.data.type === 'files_local' || source.data.type === 'files_s3') { + setCreatedSource(source); + setCurrentView('upload'); + } else { + // For other types (text, urls), close the modal + onDataSourceAdded?.(); + onClose(); + } + }; + + const handleFilesUploaded = () => { + // Just refresh the data sources list, don't close the modal + // User can continue uploading more files or close manually + onDataSourceAdded?.(); + }; + + const handleModalClose = () => { + setCurrentView('form'); + setCreatedSource(null); + onClose(); + }; + + // Reset view when modal opens + useEffect(() => { + if (isOpen) { + setCurrentView('form'); + setCreatedSource(null); + } + }, [isOpen]); + + return ( + + + +

+ {currentView === 'form' ? 'Add data source' : 'Upload files'} +

+ + + {currentView === 'form' ? ( +
+ ) : ( + createdSource && ( + + ) + )} + + {currentView === 'upload' && ( + + + + )} + + + ); +} \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/workflow/entity_list.tsx b/apps/rowboat/app/projects/[projectId]/workflow/entity_list.tsx index 76b49c11..db85b2fa 100644 --- a/apps/rowboat/app/projects/[projectId]/workflow/entity_list.tsx +++ b/apps/rowboat/app/projects/[projectId]/workflow/entity_list.tsx @@ -1,10 +1,12 @@ -import React from "react"; +import React, { forwardRef, useImperativeHandle } from "react"; import { z } from "zod"; import { WorkflowPrompt, WorkflowAgent, WorkflowTool, Workflow } from "../../../lib/types/workflow_types"; import { Project } from "../../../lib/types/project_types"; +import { DataSource } from "../../../lib/types/datasource_types"; +import { WithStringId } from "../../../lib/types/types"; import { Dropdown, DropdownItem, DropdownTrigger, DropdownMenu } from "@heroui/react"; import { useRef, useEffect, useState } from "react"; -import { EllipsisVerticalIcon, ImportIcon, PlusIcon, Brain, Boxes, Wrench, PenLine, Library, ChevronDown, ChevronRight, ServerIcon, Component, ScrollText, GripVertical, Users, Cog, CheckCircle2, LinkIcon, UnlinkIcon, MoreVertical, Eye, Trash2, AlertTriangle, Circle } from "lucide-react"; +import { EllipsisVerticalIcon, ImportIcon, PlusIcon, Brain, Boxes, Wrench, PenLine, Library, ChevronDown, ChevronRight, ServerIcon, Component, ScrollText, GripVertical, Users, Cog, CheckCircle2, LinkIcon, UnlinkIcon, MoreVertical, Eye, Trash2, AlertTriangle, Circle, Database } from "lucide-react"; import { Tooltip } from "@heroui/react"; import { DndContext, DragEndEvent, closestCenter, KeyboardSensor, PointerSensor, useSensor, useSensors } from '@dnd-kit/core'; import { SortableContext, sortableKeyboardCoordinates, useSortable, verticalListSortingStrategy } from '@dnd-kit/sortable'; @@ -17,6 +19,9 @@ import { ResizablePanelGroup, ResizablePanel, ResizableHandle } from "@/componen import { ServerLogo } from '../tools/components/MCPServersCommon'; import { Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@heroui/react"; import { ToolsModal } from './components/ToolsModal'; +import { DataSourcesModal } from './components/DataSourcesModal'; +import { DataSourceIcon } from '../../../lib/components/datasource-icon'; +import { deleteDataSource } from '../../../actions/datasource_actions'; import { ToolkitAuthModal } from '../tools/components/ToolkitAuthModal'; import { deleteConnectedAccount } from '@/app/actions/composio_actions'; import { ProjectWideChangeConfirmationModal } from '@/components/common/project-wide-change-confirmation-modal'; @@ -41,15 +46,17 @@ interface EntityListProps { agents: z.infer[]; tools: z.infer[]; prompts: z.infer[]; + dataSources: WithStringId>[]; workflow: z.infer; selectedEntity: { - type: "agent" | "tool" | "prompt" | "visualise"; + type: "agent" | "tool" | "prompt" | "datasource" | "visualise"; name: string; } | null; startAgentName: string | null; onSelectAgent: (name: string) => void; onSelectTool: (name: string) => void; onSelectPrompt: (name: string) => void; + onSelectDataSource?: (id: string) => void; onAddAgent: (agent: Partial>) => void; onAddTool: (tool: Partial>) => void; onAddPrompt: (prompt: Partial>) => void; @@ -60,7 +67,11 @@ interface EntityListProps { onDeletePrompt: (name: string) => void; onShowVisualise: (name: string) => void; onProjectToolsUpdated?: () => void; + onDataSourcesUpdated?: () => void; projectConfig?: z.infer; + useRagUploads: boolean; + useRagS3Uploads: boolean; + useRagScraping: boolean; } interface EmptyStateProps { @@ -163,7 +174,7 @@ interface ServerCardProps { serverName: string; tools: z.infer[]; selectedEntity: { - type: "agent" | "tool" | "prompt" | "visualise"; + type: "agent" | "tool" | "prompt" | "datasource" | "visualise"; name: string; } | null; onSelectTool: (name: string) => void; @@ -247,16 +258,24 @@ type ComposioToolkit = { tools: z.infer[]; } -export function EntityList({ +export const EntityList = forwardRef< + { openDataSourcesModal: () => void }, + EntityListProps & { + projectId: string, + onReorderAgents: (agents: z.infer[]) => void + } +>(function EntityList({ agents, tools, prompts, + dataSources, workflow, selectedEntity, startAgentName, onSelectAgent, onSelectTool, onSelectPrompt, + onSelectDataSource, onAddAgent, onAddTool, onAddPrompt, @@ -266,16 +285,21 @@ export function EntityList({ onDeleteTool, onDeletePrompt, onProjectToolsUpdated, + onDataSourcesUpdated, projectId, projectConfig, onReorderAgents, onShowVisualise, + useRagUploads, + useRagS3Uploads, + useRagScraping, }: EntityListProps & { projectId: string, onReorderAgents: (agents: z.infer[]) => void -}) { +}, ref) { const [showAgentTypeModal, setShowAgentTypeModal] = useState(false); const [showToolsModal, setShowToolsModal] = useState(false); + const [showDataSourcesModal, setShowDataSourcesModal] = useState(false); // State to track which toolkit's tools panel to open const [selectedToolkitSlug, setSelectedToolkitSlug] = useState(null); @@ -308,18 +332,20 @@ export function EntityList({ const [expandedPanels, setExpandedPanels] = useState({ agents: true, tools: true, + data: true, prompts: false }); // Default sizes when panels are expanded const DEFAULT_SIZES = { - agents: 40, - tools: 40, + agents: 30, + tools: 30, + data: 20, prompts: 20 }; // Calculate panel sizes based on expanded state - const getPanelSize = (panelName: 'agents' | 'tools' | 'prompts') => { + const getPanelSize = (panelName: 'agents' | 'tools' | 'data' | 'prompts') => { if (!expandedPanels[panelName]) { return 8; // Collapsed height (53px equivalent) } @@ -332,13 +358,23 @@ export function EntityList({ if (!expandedPanels.tools) { size += DEFAULT_SIZES.tools; } + if (!expandedPanels.data) { + size += DEFAULT_SIZES.data; + } if (!expandedPanels.prompts) { size += DEFAULT_SIZES.prompts; } } else if (panelName === 'tools') { + if (!expandedPanels.data && expandedPanels.agents) { + size += DEFAULT_SIZES.data; + } if (!expandedPanels.prompts && expandedPanels.agents) { size += DEFAULT_SIZES.prompts; } + } else if (panelName === 'data') { + if (!expandedPanels.prompts && (expandedPanels.agents || expandedPanels.tools)) { + size += DEFAULT_SIZES.prompts; + } } return size; @@ -366,6 +402,10 @@ export function EntityList({ onSelectTool(name); } + function handleSelectDataSource(id: string) { + onSelectDataSource?.(id); + } + const sensors = useSensors( useSensor(PointerSensor), useSensor(KeyboardSensor, { @@ -394,6 +434,12 @@ export function EntityList({ } }; + useImperativeHandle(ref, () => ({ + openDataSourcesModal: () => { + setShowDataSourcesModal(true); + } + })); + return (
+ {/* Data Panel */} + + +
+ + + Data +
+
+ +
+
+ } + > + {expandedPanels.data && ( +
+
+ {dataSources.length > 0 ? ( +
+ {dataSources.map((dataSource, index) => { + // Determine data source status + const isActive = dataSource.active && dataSource.status === 'ready'; + const isPending = dataSource.status === 'pending'; + const isError = dataSource.status === 'error'; + + let statusPill = null; + if (isPending) { + statusPill = ( + + + + Processing + + + ); + } else if (isError) { + statusPill = ( + + + + Error + + + ); + } else if (isActive) { + statusPill = ( + + + + Active + + + ); + } else { + statusPill = ( + + + + Inactive + + + ); + } + + return ( +
+
+ +
+ {statusPill} +
+ { + if (window.confirm(`Are you sure you want to delete the data source "${dataSource.name}"?`)) { + await deleteDataSource(projectId, dataSource._id); + onDataSourcesUpdated?.(); + } + }} + /> +
+
+
+
+ ); + })} +
+ ) : ( + + )} +
+
+ )} + + + + + {/* Prompts Panel */} + setShowDataSourcesModal(false)} + projectId={projectId} + onDataSourceAdded={onDataSourcesUpdated} + useRagUploads={useRagUploads} + useRagS3Uploads={useRagS3Uploads} + useRagScraping={useRagScraping} + />
); -} +}); function AgentDropdown({ agent, @@ -823,7 +1027,7 @@ function EntityDropdown({ interface ComposioCardProps { card: ComposioToolkit; selectedEntity: { - type: "agent" | "tool" | "prompt" | "visualise"; + type: "agent" | "tool" | "prompt" | "datasource" | "visualise"; name: string; } | null; onSelectTool: (name: string) => void; diff --git a/apps/rowboat/app/projects/[projectId]/workflow/page.tsx b/apps/rowboat/app/projects/[projectId]/workflow/page.tsx index 144eaf66..224bc223 100644 --- a/apps/rowboat/app/projects/[projectId]/workflow/page.tsx +++ b/apps/rowboat/app/projects/[projectId]/workflow/page.tsx @@ -1,6 +1,6 @@ import { Metadata } from "next"; import { App } from "./app"; -import { USE_RAG } from "@/app/lib/feature_flags"; +import { USE_RAG, USE_RAG_UPLOADS, USE_RAG_S3_UPLOADS, USE_RAG_SCRAPING } from "@/app/lib/feature_flags"; import { projectsCollection } from "@/app/lib/mongodb"; import { notFound } from "next/navigation"; import { requireActiveBillingSubscription } from '@/app/lib/billing'; @@ -38,6 +38,9 @@ export default async function Page( ); diff --git a/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx b/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx index 98fa27ec..46688a8f 100644 --- a/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx +++ b/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx @@ -11,7 +11,7 @@ import { App as ChatApp } from "../playground/app"; import { z } from "zod"; import { Button, Dropdown, DropdownItem, DropdownMenu, DropdownTrigger, Spinner, Tooltip, Modal, ModalContent, ModalHeader, ModalBody, ModalFooter, useDisclosure } from "@heroui/react"; import { PromptConfig } from "../entities/prompt_config"; -import { InputField } from "../../../lib/components/input-field"; +import { DataSourceConfig } from "../entities/datasource_config"; import { RelativeTime } from "@primer/react"; import { USE_PRODUCT_TOUR } from "@/app/lib/feature_flags"; @@ -44,7 +44,7 @@ interface StateItem { workflow: z.infer; publishing: boolean; selection: { - type: "agent" | "tool" | "prompt" | "visualise"; + type: "agent" | "tool" | "prompt" | "datasource" | "visualise"; name: string; } | null; saving: boolean; @@ -139,6 +139,11 @@ export type Action = { } | { type: "reorder_agents"; agents: z.infer[]; +} | { + type: "select_datasource"; + id: string; +} | { + type: "unselect_datasource"; } | { type: "show_visualise"; } | { @@ -260,9 +265,16 @@ function reducer(state: State, action: Action): State { name: action.name }; break; + case "select_datasource": + draft.selection = { + type: "datasource", + name: action.id + }; + break; case "unselect_agent": case "unselect_tool": case "unselect_prompt": + case "unselect_datasource": draft.selection = null; break; case "add_agent": { @@ -575,6 +587,9 @@ export function WorkflowEditor({ dataSources, workflow, useRag, + useRagUploads, + useRagS3Uploads, + useRagScraping, mcpServerUrls, defaultModel, projectConfig, @@ -583,11 +598,15 @@ export function WorkflowEditor({ onChangeMode, onRevertToLive, onProjectToolsUpdated, + onDataSourcesUpdated, }: { projectId: string; dataSources: WithStringId>[]; workflow: z.infer; useRag: boolean; + useRagUploads: boolean; + useRagS3Uploads: boolean; + useRagScraping: boolean; mcpServerUrls: Array>; defaultModel: string; projectConfig: z.infer; @@ -596,6 +615,7 @@ export function WorkflowEditor({ onChangeMode: (mode: 'draft' | 'live') => void; onRevertToLive: () => void; onProjectToolsUpdated?: () => void; + onDataSourcesUpdated?: () => void; }) { const [state, dispatch] = useReducer(reducer, { @@ -628,6 +648,7 @@ export function WorkflowEditor({ const [isInitialState, setIsInitialState] = useState(true); const [showTour, setShowTour] = useState(true); const copilotRef = useRef<{ handleUserMessage: (message: string) => void }>(null); + const entityListRef = useRef<{ openDataSourcesModal: () => void } | null>(null); // Modal state for revert confirmation const { isOpen: isRevertModalOpen, onOpen: onRevertModalOpen, onClose: onRevertModalClose } = useDisclosure(); @@ -662,6 +683,10 @@ export function WorkflowEditor({ }, 100); }, []); + const handleOpenDataSourcesModal = useCallback(() => { + entityListRef.current?.openDataSourcesModal(); + }, []); + console.log(`workflow editor chat key: ${state.present.chatKey}`); // Auto-show copilot and increment key when prompt is present @@ -698,6 +723,9 @@ export function WorkflowEditor({ function handleSelectPrompt(name: string) { dispatch({ type: "select_prompt", name }); } + function handleSelectDataSource(id: string) { + dispatch({ type: "select_datasource", id }); + } function handleUnselectAgent() { dispatch({ type: "unselect_agent" }); @@ -977,15 +1005,18 @@ export function WorkflowEditor({
@@ -1041,6 +1077,7 @@ export function WorkflowEditor({ useRag={useRag} triggerCopilotChat={triggerCopilotChat} eligibleModels={eligibleModels === "*" ? "*" : eligibleModels.agentModels} + onOpenDataSourcesModal={handleOpenDataSourcesModal} />} {state.present.selection?.type === "tool" && (() => { const selectedTool = state.present.workflow.tools.find( @@ -1066,6 +1103,12 @@ export function WorkflowEditor({ handleUpdate={handleUpdatePrompt.bind(null, state.present.selection.name)} handleClose={handleUnselectPrompt} />} + {state.present.selection?.type === "datasource" && dispatch({ type: "unselect_datasource" })} + onDataSourceUpdate={onDataSourcesUpdated} + />} {state.present.selection?.type === "visualise" && ( ) { return ( - + {children} ); diff --git a/apps/rowboat/app/projects/layout/components/app-layout.tsx b/apps/rowboat/app/projects/layout/components/app-layout.tsx index 38fc77f2..5bc6e25d 100644 --- a/apps/rowboat/app/projects/layout/components/app-layout.tsx +++ b/apps/rowboat/app/projects/layout/components/app-layout.tsx @@ -8,12 +8,11 @@ import { useRouter } from 'next/navigation'; interface AppLayoutProps { children: ReactNode; - useRag?: boolean; useAuth?: boolean; useBilling?: boolean; } -export default function AppLayout({ children, useRag = false, useAuth = false, useBilling = false }: AppLayoutProps) { +export default function AppLayout({ children, useAuth = false, useBilling = false }: AppLayoutProps) { const router = useRouter(); const [sidebarCollapsed, setSidebarCollapsed] = useState(true); const [billingPastDue, setBillingPastDue] = useState(false); @@ -46,7 +45,6 @@ export default function AppLayout({ children, useRag = false, useAuth = false, u
setSidebarCollapsed(!sidebarCollapsed)} diff --git a/apps/rowboat/app/projects/layout/components/sidebar.tsx b/apps/rowboat/app/projects/layout/components/sidebar.tsx index 027cc8c3..5f6cf324 100644 --- a/apps/rowboat/app/projects/layout/components/sidebar.tsx +++ b/apps/rowboat/app/projects/layout/components/sidebar.tsx @@ -5,7 +5,6 @@ import { usePathname } from "next/navigation"; import { Tooltip } from "@heroui/react"; import { UserButton } from "@/app/lib/components/user_button"; import { - DatabaseIcon, SettingsIcon, WorkflowIcon, PlayIcon, @@ -23,7 +22,6 @@ import { useHelpModal } from "@/app/providers/help-modal-provider"; interface SidebarProps { projectId?: string; - useRag: boolean; useAuth: boolean; collapsed?: boolean; onToggleCollapse?: () => void; @@ -33,7 +31,7 @@ interface SidebarProps { const EXPANDED_ICON_SIZE = 20; const COLLAPSED_ICON_SIZE = 20; // DO NOT CHANGE THIS -export default function Sidebar({ projectId, useRag, useAuth, collapsed = false, onToggleCollapse, useBilling }: SidebarProps) { +export default function Sidebar({ projectId, useAuth, collapsed = false, onToggleCollapse, useBilling }: SidebarProps) { const pathname = usePathname(); const [projectName, setProjectName] = useState("Select Project"); const isProjectsRoute = pathname === '/projects'; @@ -62,12 +60,6 @@ export default function Sidebar({ projectId, useRag, useAuth, collapsed = false, icon: WorkflowIcon, requiresProject: true }, - ...(useRag ? [{ - href: 'sources', - label: 'RAG', - icon: DatabaseIcon, - requiresProject: true - }] : []), { href: 'config', label: 'Settings', diff --git a/apps/rowboat/app/projects/layout/menu.tsx b/apps/rowboat/app/projects/layout/menu.tsx index 00c87321..5b92b5ac 100644 --- a/apps/rowboat/app/projects/layout/menu.tsx +++ b/apps/rowboat/app/projects/layout/menu.tsx @@ -1,7 +1,7 @@ 'use client'; import { usePathname } from "next/navigation"; import Link from "next/link"; -import { DatabaseIcon, SettingsIcon, WorkflowIcon, PlayIcon, LucideIcon } from "lucide-react"; +import { SettingsIcon, WorkflowIcon, PlayIcon, LucideIcon } from "lucide-react"; import MenuItem from "./components/menu-item"; interface NavLinkProps { @@ -29,11 +29,9 @@ function NavLink({ href, label, icon, collapsed, selected = false }: NavLinkProp export default function Menu({ projectId, collapsed, - useRag, }: { projectId: string; collapsed: boolean; - useRag: boolean; }) { const pathname = usePathname(); @@ -53,15 +51,6 @@ export default function Menu({ icon={PlayIcon} selected={pathname.startsWith(`/projects/${projectId}/test`)} /> - {useRag && ( - - )} (null); @@ -56,6 +54,6 @@ export function Nav({ } - +
; } \ No newline at end of file