apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-05-25 18:55:19 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

DDD refactor: data-sources (#205)

Browse source
This commit is contained in:
Ramnique Singh 2025-08-17 08:06:17 +05:30 • committed by GitHub
parent 912c8be156
commit 4b33b20e76
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
68 changed files with 2589 additions and 1588 deletions
Download patch file Download diff file Expand all files Collapse all files

7
apps/rowboat/app/projects/[projectId]/copilot/app.tsx
View file

@ -5,7 +5,7 @@ import { useRef, useState, createContext, useContext, useCallback, forwardRef, u
import { CopilotChatContext } from "../../../lib/types/copilot_types";
import { CopilotMessage } from "../../../lib/types/copilot_types";
import { Workflow } from "@/app/lib/types/workflow_types";
import { DataSource } from "@/app/lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { Action as WorkflowDispatch } from "@/app/projects/[projectId]/workflow/workflow_editor";
import { Panel } from "@/components/common/panel-common";
@ -14,7 +14,6 @@ import { Messages } from "./components/messages";
import { CopyIcon, CheckIcon, PlusIcon, XIcon, InfoIcon, Sparkles } from "lucide-react";
import { useCopilot } from "./use-copilot";
import { BillingUpgradeModal } from "@/components/common/billing-upgrade-modal";
import { WithStringId } from "@/app/lib/types/types";
const CopilotContext = createContext<{
workflow: z.infer<typeof Workflow> | null;
@ -33,7 +32,7 @@ interface AppProps {
onCopyJson?: (data: { messages: any[] }) => void;
onMessagesChange?: (messages: z.infer<typeof CopilotMessage>[]) => void;
isInitialState?: boolean;
dataSources?: WithStringId<z.infer<typeof DataSource>>[];
dataSources?: z.infer<typeof DataSource>[];
}
const App = forwardRef<{ handleCopyChat: () => void; handleUserMessage: (message: string) => void }, AppProps>(function App({
@ -277,7 +276,7 @@ export const Copilot = forwardRef<{ handleUserMessage: (message: string) => void
chatContext?: z.infer<typeof CopilotChatContext>;
dispatch: (action: WorkflowDispatch) => void;
isInitialState?: boolean;
dataSources?: WithStringId<z.infer<typeof DataSource>>[];
dataSources?: z.infer<typeof DataSource>[];
}>(({
projectId,
workflow,

4
apps/rowboat/app/projects/[projectId]/copilot/use-copilot.tsx
View file

@ -2,7 +2,7 @@ import { useCallback, useRef, useState } from "react";
import { getCopilotResponseStream } from "@/app/actions/copilot.actions";
import { CopilotMessage } from "@/app/lib/types/copilot_types";
import { Workflow } from "@/app/lib/types/workflow_types";
import { DataSource } from "@/app/lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { WithStringId } from "@/app/lib/types/types";
@ -10,7 +10,7 @@ interface UseCopilotParams {
projectId: string;
workflow: z.infer<typeof Workflow>;
context: any;
dataSources?: WithStringId<z.infer<typeof DataSource>>[];
dataSources?: z.infer<typeof DataSource>[];
}
interface UseCopilotResult {

13
apps/rowboat/app/projects/[projectId]/entities/agent_config.tsx
View file

@ -1,7 +1,6 @@
"use client";
import { WithStringId } from "../../../lib/types/types";
import { WorkflowPrompt, WorkflowAgent, Workflow, WorkflowTool } from "../../../lib/types/workflow_types";
import { DataSource } from "../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { PlusIcon, Sparkles, X as XIcon, ChevronDown, ChevronRight, Trash2, Maximize2, Minimize2, StarIcon, DatabaseIcon, UserIcon, Settings, Info } from "lucide-react";
import { useState, useEffect, useRef } from "react";
@ -59,7 +58,7 @@ export function AgentConfig({
agents: z.infer<typeof WorkflowAgent>[],
tools: z.infer<typeof WorkflowTool>[],
prompts: z.infer<typeof WorkflowPrompt>[],
dataSources: WithStringId<z.infer<typeof DataSource>>[],
dataSources: z.infer<typeof DataSource>[],
handleUpdate: (agent: z.infer<typeof WorkflowAgent>) => void,
handleClose: () => void,
useRag: boolean,
@ -726,12 +725,12 @@ export function AgentConfig({
startContent={<PlusIcon className="w-4 h-4 text-gray-500" />}
>
{dataSources
.filter((ds) => !(agent.ragDataSources || []).includes(ds._id))
.filter((ds) => !(agent.ragDataSources || []).includes(ds.id))
.length > 0 ? (
dataSources
.filter((ds) => !(agent.ragDataSources || []).includes(ds._id))
.filter((ds) => !(agent.ragDataSources || []).includes(ds.id))
.map((ds) => (
<SelectItem key={ds._id}>
<SelectItem key={ds.id}>
{ds.name}
</SelectItem>
))
@ -775,7 +774,7 @@ export function AgentConfig({
{agent.ragDataSources !== undefined && agent.ragDataSources.length > 0 && (
<div className="flex flex-col gap-2 mt-2">
{(agent.ragDataSources || []).map((source) => {
const ds = dataSources.find((ds) => ds._id === source);
const ds = dataSources.find((ds) => ds.id === source);
return (
<div
key={source}

87
apps/rowboat/app/projects/[projectId]/entities/datasource_config.tsx
View file

@ -1,6 +1,5 @@
"use client";
import { WithStringId } from "../../../lib/types/types";
import { DataSource } from "../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { XIcon, FileIcon, GlobeIcon, AlertTriangle, CheckCircle, Circle, ExternalLinkIcon, Type, PlusIcon, Edit3Icon, DownloadIcon, Trash2 } from "lucide-react";
import { useState, useEffect, useCallback } from "react";
@ -8,9 +7,9 @@ import { Panel } from "@/components/common/panel-common";
import { Button } from "@/components/ui/button";
import { DataSourceIcon } from "@/app/lib/components/datasource-icon";
import { Tooltip } from "@heroui/react";
import { getDataSource, listDocsInDataSource, deleteDocsFromDataSource, getDownloadUrlForFile, addDocsToDataSource, getUploadUrlsForFilesDataSource } from "@/app/actions/data-source.actions";
import { getDataSource, listDocsInDataSource, deleteDocFromDataSource, getDownloadUrlForFile, addDocsToDataSource, getUploadUrlsForFilesDataSource } from "@/app/actions/data-source.actions";
import { InputField } from "@/app/lib/components/input-field";
import { DataSourceDoc } from "../../../lib/types/datasource_types";
import { DataSourceDoc } from "@/src/entities/models/data-source-doc";
import { RelativeTime } from "@primer/react";
import { Pagination, Spinner, Button as HeroButton, Textarea as HeroTextarea } from "@heroui/react";
import { useDropzone } from "react-dropzone";
@ -24,12 +23,12 @@ export function DataSourceConfig({
handleClose: () => void,
onDataSourceUpdate?: () => void
}) {
const [dataSource, setDataSource] = useState<WithStringId<z.infer<typeof DataSource>> | null>(null);
const [dataSource, setDataSource] = useState<z.infer<typeof DataSource> | null>(null);
const [loading, setLoading] = useState(true);
const [error, setError] = useState<string | null>(null);
// Files-related state
const [files, setFiles] = useState<WithStringId<z.infer<typeof DataSourceDoc>>[]>([]);
const [files, setFiles] = useState<z.infer<typeof DataSourceDoc>[]>([]);
const [filesLoading, setFilesLoading] = useState(false);
const [filesPage, setFilesPage] = useState(1);
const [filesTotal, setFilesTotal] = useState(0);
@ -44,22 +43,22 @@ export function DataSourceConfig({
const currentProjectId = pathParts[2]; // /projects/[projectId]/workflow
setProjectId(currentProjectId);
const ds = await getDataSource(currentProjectId, dataSourceId);
const ds = await getDataSource(dataSourceId);
setDataSource(ds);
// Load files if it's a files data source
if (ds.data.type === 'files_local' || ds.data.type === 'files_s3') {
await loadFiles(currentProjectId, dataSourceId, 1);
await loadFiles(dataSourceId, 1);
}
// Load URLs if it's a URLs data source
if (ds.data.type === 'urls') {
await loadUrls(currentProjectId, dataSourceId, 1);
await loadUrls(dataSourceId, 1);
}
// Load text content if it's a text data source
if (ds.data.type === 'text') {
await loadTextContent(currentProjectId, dataSourceId);
await loadTextContent(dataSourceId);
}
} catch (err) {
console.error('Failed to load data source:', err);
@ -91,7 +90,7 @@ export function DataSourceConfig({
}
try {
const updatedSource = await getDataSource(projectId, dataSourceId);
const updatedSource = await getDataSource(dataSourceId);
if (!ignore) {
setDataSource(updatedSource);
onDataSourceUpdate?.(); // Notify parent of status change
@ -124,20 +123,19 @@ export function DataSourceConfig({
// Helper function to update data source and notify parent
const updateDataSourceAndNotify = useCallback(async () => {
try {
const updatedSource = await getDataSource(projectId, dataSourceId);
const updatedSource = await getDataSource(dataSourceId);
setDataSource(updatedSource);
onDataSourceUpdate?.();
} catch (err) {
console.error('Failed to reload data source:', err);
}
}, [projectId, dataSourceId, onDataSourceUpdate]);
}, [dataSourceId, onDataSourceUpdate]);
// Load files function
const loadFiles = async (projectId: string, sourceId: string, page: number) => {
const loadFiles = async (sourceId: string, page: number) => {
try {
setFilesLoading(true);
const { files, total } = await listDocsInDataSource({
projectId,
sourceId,
page,
limit: 10,
@ -153,7 +151,7 @@ export function DataSourceConfig({
};
// URLs-related state
const [urls, setUrls] = useState<WithStringId<z.infer<typeof DataSourceDoc>>[]>([]);
const [urls, setUrls] = useState<z.infer<typeof DataSourceDoc>[]>([]);
const [urlsLoading, setUrlsLoading] = useState(false);
const [urlsPage, setUrlsPage] = useState(1);
const [urlsTotal, setUrlsTotal] = useState(0);
@ -171,11 +169,10 @@ export function DataSourceConfig({
const [uploadingFiles, setUploadingFiles] = useState(false);
// Load URLs function
const loadUrls = async (projectId: string, sourceId: string, page: number) => {
const loadUrls = async (sourceId: string, page: number) => {
try {
setUrlsLoading(true);
const { files, total } = await listDocsInDataSource({
projectId,
sourceId,
page,
limit: 10,
@ -191,11 +188,10 @@ export function DataSourceConfig({
};
// Load text content function
const loadTextContent = async (projectId: string, sourceId: string) => {
const loadTextContent = async (sourceId: string) => {
try {
setTextLoading(true);
const { files } = await listDocsInDataSource({
projectId,
sourceId,
limit: 1,
});
@ -218,13 +214,11 @@ export function DataSourceConfig({
if (!window.confirm('Are you sure you want to delete this file?')) return;
try {
await deleteDocsFromDataSource({
projectId,
sourceId: dataSourceId,
docIds: [fileId],
await deleteDocFromDataSource({
docId: fileId,
});
// Reload files
await loadFiles(projectId, dataSourceId, filesPage);
await loadFiles(dataSourceId, filesPage);
// Reload data source to get updated status
await updateDataSourceAndNotify();
@ -236,7 +230,7 @@ export function DataSourceConfig({
// Handle file download
const handleDownloadFile = async (fileId: string) => {
try {
const url = await getDownloadUrlForFile(projectId, dataSourceId, fileId);
const url = await getDownloadUrlForFile(fileId);
window.open(url, '_blank');
} catch (err) {
console.error('Failed to download file:', err);
@ -245,7 +239,7 @@ export function DataSourceConfig({
// Handle page change
const handlePageChange = (page: number) => {
loadFiles(projectId, dataSourceId, page);
loadFiles(dataSourceId, page);
};
// Handle URL deletion
@ -253,13 +247,11 @@ export function DataSourceConfig({
if (!window.confirm('Are you sure you want to delete this URL?')) return;
try {
await deleteDocsFromDataSource({
projectId,
sourceId: dataSourceId,
docIds: [urlId],
await deleteDocFromDataSource({
docId: urlId,
});
// Reload URLs
await loadUrls(projectId, dataSourceId, urlsPage);
await loadUrls(dataSourceId, urlsPage);
// Reload data source to get updated status
await updateDataSourceAndNotify();
@ -270,7 +262,7 @@ export function DataSourceConfig({
// Handle URL page change
const handleUrlPageChange = (page: number) => {
loadUrls(projectId, dataSourceId, page);
loadUrls(dataSourceId, page);
};
// Handle text content update
@ -279,22 +271,18 @@ export function DataSourceConfig({
try {
// Delete existing text doc if it exists
const { files } = await listDocsInDataSource({
projectId,
sourceId: dataSourceId,
limit: 1,
});
if (files.length > 0) {
await deleteDocsFromDataSource({
projectId,
sourceId: dataSourceId,
docIds: [files[0]._id],
await deleteDocFromDataSource({
docId: files[0].id,
});
}
// Add new text doc
await addDocsToDataSource({
projectId,
sourceId: dataSourceId,
docData: [{
name: 'text',
@ -327,7 +315,6 @@ export function DataSourceConfig({
const first100Urls = urlsArray.slice(0, 100);
await addDocsToDataSource({
projectId,
sourceId: dataSourceId,
docData: first100Urls.map(url => ({
name: url,
@ -339,7 +326,7 @@ export function DataSourceConfig({
});
setShowAddUrlForm(false);
await loadUrls(projectId, dataSourceId, urlsPage);
await loadUrls(dataSourceId, urlsPage);
// Reload data source to get updated status
await updateDataSourceAndNotify();
@ -356,7 +343,7 @@ export function DataSourceConfig({
setUploadingFiles(true);
try {
const urls = await getUploadUrlsForFilesDataSource(projectId, dataSourceId, acceptedFiles.map(file => ({
const urls = await getUploadUrlsForFilesDataSource(dataSourceId, acceptedFiles.map(file => ({
name: file.name,
type: file.type,
size: file.size,
@ -403,17 +390,17 @@ export function DataSourceConfig({
name: file.name,
size: file.size,
mimeType: file.type,
path: urls[index].path,
},
}));
}
await addDocsToDataSource({
projectId,
sourceId: dataSourceId,
docData,
});
await loadFiles(projectId, dataSourceId, filesPage);
await loadFiles(dataSourceId, filesPage);
// Reload data source to get updated status
await updateDataSourceAndNotify();
@ -422,7 +409,7 @@ export function DataSourceConfig({
} finally {
setUploadingFiles(false);
}
}, [projectId, dataSourceId, dataSource, filesPage, updateDataSourceAndNotify]);
}, [dataSourceId, dataSource, filesPage, updateDataSourceAndNotify]);
const { getRootProps, getInputProps, isDragActive } = useDropzone({
onDrop: onFileDrop,
@ -676,7 +663,7 @@ export function DataSourceConfig({
<div className="space-y-2">
{files.map((file) => (
<div
key={file._id}
key={file.id}
className="flex items-center justify-between p-3 bg-gray-50 dark:bg-gray-800/50 rounded-lg border"
>
<div className="flex items-center gap-3 flex-1 min-w-0">
@ -696,7 +683,7 @@ export function DataSourceConfig({
{(file.data.type === 'file_local' || file.data.type === 'file_s3') && (
<Tooltip content="Download file">
<button
onClick={() => handleDownloadFile(file._id)}
onClick={() => handleDownloadFile(file.id)}
className="p-1 hover:bg-gray-200 dark:hover:bg-gray-700 rounded transition-colors"
>
<DownloadIcon className="w-4 h-4 text-gray-500" />
@ -705,7 +692,7 @@ export function DataSourceConfig({
)}
<Tooltip content="Delete file">
<button
onClick={() => handleDeleteFile(file._id)}
onClick={() => handleDeleteFile(file.id)}
className="p-1 hover:bg-red-100 dark:hover:bg-red-900/20 rounded transition-colors"
>
<Trash2 className="w-4 h-4 text-red-500" />
@ -805,7 +792,7 @@ export function DataSourceConfig({
<div className="space-y-2">
{urls.map((url) => (
<div
key={url._id}
key={url.id}
className="flex items-center justify-between p-3 bg-gray-50 dark:bg-gray-800/50 rounded-lg border"
>
<div className="flex items-center gap-3 flex-1 min-w-0">
@ -834,7 +821,7 @@ export function DataSourceConfig({
<div className="flex items-center gap-2">
<Tooltip content="Delete URL">
<button
onClick={() => handleDeleteUrl(url._id)}
onClick={() => handleDeleteUrl(url.id)}
className="p-1 hover:bg-red-100 dark:hover:bg-red-900/20 rounded transition-colors"
>
<Trash2 className="w-4 h-4 text-red-500" />

22
apps/rowboat/app/projects/[projectId]/sources/[sourceId]/source-page.tsx
View file

@ -1,6 +1,5 @@
'use client';
import { WithStringId } from "../../../../lib/types/types";
import { DataSource } from "../../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { ToggleSource } from "../components/toggle-source";
import { Spinner } from "@heroui/react";
import { SourceStatus } from "../components/source-status";
@ -28,14 +27,14 @@ export function SourcePage({
sourceId: string;
projectId: string;
}) {
const [source, setSource] = useState<WithStringId<z.infer<typeof DataSource>> | null>(null);
const [source, setSource] = useState<z.infer<typeof DataSource> | null>(null);
const [isLoading, setIsLoading] = useState(true);
const [showSaveSuccess, setShowSaveSuccess] = useState(false);
const [billingError, setBillingError] = useState<string | null>(null);
async function handleReload() {
setIsLoading(true);
const updatedSource = await getDataSource(projectId, sourceId);
const updatedSource = await getDataSource(sourceId);
setSource(updatedSource);
if ("billingError" in updatedSource && updatedSource.billingError) {
setBillingError(updatedSource.billingError);
@ -48,7 +47,7 @@ export function SourcePage({
let ignore = false;
async function fetchSource() {
setIsLoading(true);
const source = await getDataSource(projectId, sourceId);
const source = await getDataSource(sourceId);
if (!ignore) {
setSource(source);
if ("billingError" in source && source.billingError) {
@ -61,7 +60,7 @@ export function SourcePage({
return () => {
ignore = true;
};
}, [projectId, sourceId]);
}, [sourceId]);
// refresh source data every 15 seconds
// under certain conditions
@ -80,7 +79,7 @@ export function SourcePage({
if (timeout) {
clearTimeout(timeout);
}
const updatedSource = await getDataSource(projectId, sourceId);
const updatedSource = await getDataSource(sourceId);
if (!ignore) {
setSource(updatedSource);
if ("billingError" in updatedSource && updatedSource.billingError) {
@ -130,7 +129,6 @@ export function SourcePage({
<SectionLabel>Toggle</SectionLabel>
<SectionContent>
<ToggleSource
projectId={projectId}
sourceId={sourceId}
active={source.active}
/>
@ -153,7 +151,6 @@ export function SourcePage({
action={async (formData: FormData) => {
const description = formData.get('description') as string;
await updateDataSource({
projectId,
sourceId,
description,
});
@ -217,7 +214,7 @@ export function SourcePage({
<SectionRow>
<SectionLabel>Status</SectionLabel>
<SectionContent>
<SourceStatus status={source.status} projectId={projectId} />
<SourceStatus status={source.status} />
{("billingError" in source) && source.billingError && <div className="flex flex-col gap-1 items-start mt-4">
<div className="text-sm">{source.billingError}</div>
@ -240,14 +237,12 @@ export function SourcePage({
{/* Source-specific sections */}
{source.data.type === 'urls' &&
<ScrapeSource
projectId={projectId}
dataSource={source}
handleReload={handleReload}
/>
}
{(source.data.type === 'files_local' || source.data.type === 'files_s3') &&
<FilesSource
projectId={projectId}
dataSource={source}
handleReload={handleReload}
type={source.data.type}
@ -255,7 +250,6 @@ export function SourcePage({
}
{source.data.type === 'text' &&
<TextSource
projectId={projectId}
dataSource={source}
handleReload={handleReload}
/>
@ -272,7 +266,7 @@ export function SourcePage({
This action cannot be undone.
</p>
</div>
<DeleteSource projectId={projectId} sourceId={sourceId} />
<DeleteSource sourceId={sourceId} />
</div>
</Section>
</div>

4
apps/rowboat/app/projects/[projectId]/sources/components/delete.tsx
View file

@ -4,15 +4,13 @@ import { deleteDataSource } from "../../../../actions/data-source.actions";
import { FormStatusButton } from "../../../../lib/components/form-status-button";
export function DeleteSource({
projectId,
sourceId,
}: {
projectId: string;
sourceId: string;
}) {
function handleDelete() {
if (window.confirm('Are you sure you want to delete this data source?')) {
deleteDataSource(projectId, sourceId);
deleteDataSource(sourceId);
}
}

48
apps/rowboat/app/projects/[projectId]/sources/components/files-source.tsx
View file

@ -1,24 +1,20 @@
"use client";
import { WithStringId } from "../../../../lib/types/types";
import { DataSourceDoc, DataSource } from "../../../../lib/types/datasource_types";
import { DataSourceDoc } from "@/src/entities/models/data-source-doc";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { useCallback, useEffect, useState } from "react";
import { useDropzone } from "react-dropzone";
import { deleteDocsFromDataSource, getUploadUrlsForFilesDataSource, addDocsToDataSource, getDownloadUrlForFile, listDocsInDataSource } from "../../../../actions/data-source.actions";
import { deleteDocFromDataSource, getUploadUrlsForFilesDataSource, addDocsToDataSource, getDownloadUrlForFile, listDocsInDataSource } from "../../../../actions/data-source.actions";
import { RelativeTime } from "@primer/react";
import { Pagination, Spinner } from "@heroui/react";
import { DownloadIcon } from "lucide-react";
import { Section } from "./section";
function FileListItem({
projectId,
sourceId,
file,
onDelete,
}: {
projectId: string,
sourceId: string,
file: WithStringId<z.infer<typeof DataSourceDoc>>,
file: z.infer<typeof DataSourceDoc>,
onDelete: (fileId: string) => Promise<void>;
}) {
const [isDeleting, setIsDeleting] = useState(false);
@ -27,7 +23,7 @@ function FileListItem({
const handleDeleteClick = async () => {
setIsDeleting(true);
try {
await onDelete(file._id);
await onDelete(file.id);
} finally {
setIsDeleting(false);
}
@ -36,7 +32,7 @@ function FileListItem({
const handleDownloadClick = async () => {
setIsDownloading(true);
try {
const url = await getDownloadUrlForFile(projectId, sourceId, file._id);
const url = await getDownloadUrlForFile(file.id);
window.open(url, '_blank');
} catch (error) {
console.error('Download failed:', error);
@ -90,17 +86,15 @@ function FileListItem({
}
function PaginatedFileList({
projectId,
sourceId,
handleReload,
onDelete,
}: {
projectId: string,
sourceId: string,
handleReload: () => void;
onDelete: (fileId: string) => Promise<void>;
}) {
const [files, setFiles] = useState<WithStringId<z.infer<typeof DataSourceDoc>>[]>([]);
const [files, setFiles] = useState<z.infer<typeof DataSourceDoc>[]>([]);
const [page, setPage] = useState(1);
const [total, setTotal] = useState(0);
const [loading, setLoading] = useState(false);
@ -114,7 +108,6 @@ function PaginatedFileList({
setLoading(true);
try {
const { files, total } = await listDocsInDataSource({
projectId,
sourceId,
page,
limit: 10,
@ -134,7 +127,7 @@ function PaginatedFileList({
return () => {
ignore = true;
}
}, [projectId, sourceId, page]);
}, [sourceId, page]);
return (
<div className="space-y-4">
@ -154,10 +147,8 @@ function PaginatedFileList({
<div className="space-y-3">
{files.map(file => (
<FileListItem
key={file._id}
key={file.id}
file={file}
projectId={projectId}
sourceId={sourceId}
onDelete={onDelete}
/>
))}
@ -177,13 +168,11 @@ function PaginatedFileList({
}
export function FilesSource({
projectId,
dataSource,
handleReload,
type,
}: {
projectId: string,
dataSource: WithStringId<z.infer<typeof DataSource>>,
dataSource: z.infer<typeof DataSource>,
handleReload: () => void;
type: 'files_local' | 'files_s3';
}) {
@ -193,7 +182,7 @@ export function FilesSource({
const onDrop = useCallback(async (acceptedFiles: File[]) => {
setUploading(true);
try {
const urls = await getUploadUrlsForFilesDataSource(projectId, dataSource._id, acceptedFiles.map(file => ({
const urls = await getUploadUrlsForFilesDataSource(dataSource.id, acceptedFiles.map(file => ({
name: file.name,
type: file.type,
size: file.size,
@ -237,13 +226,13 @@ export function FilesSource({
name: file.name,
size: file.size,
mimeType: file.type,
path: urls[index].path,
},
}));
}
await addDocsToDataSource({
projectId,
sourceId: dataSource._id,
sourceId: dataSource.id,
docData,
});
@ -255,7 +244,7 @@ export function FilesSource({
} finally {
setUploading(false);
}
}, [projectId, dataSource._id, handleReload, type]);
}, [dataSource.id, handleReload, type]);
const { getRootProps, getInputProps, isDragActive } = useDropzone({
onDrop,
@ -299,14 +288,11 @@ export function FilesSource({
<PaginatedFileList
key={fileListKey}
projectId={projectId}
sourceId={dataSource._id}
sourceId={dataSource.id}
handleReload={handleReload}
onDelete={async (docId) => {
await deleteDocsFromDataSource({
projectId,
sourceId: dataSource._id,
docIds: [docId],
await deleteDocFromDataSource({
docId: docId,
});
handleReload();
setFileListKey(prev => prev + 1);

41
apps/rowboat/app/projects/[projectId]/sources/components/scrape-source.tsx
View file

@ -1,9 +1,9 @@
"use client";
import { WithStringId } from "../../../../lib/types/types";
import { DataSourceDoc, DataSource } from "../../../../lib/types/datasource_types";
import { DataSourceDoc } from "@/src/entities/models/data-source-doc";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { Recrawl } from "./web-recrawl";
import { deleteDocsFromDataSource, listDocsInDataSource, recrawlWebDataSource, addDocsToDataSource } from "../../../../actions/data-source.actions";
import { deleteDocFromDataSource, listDocsInDataSource, recrawlWebDataSource, addDocsToDataSource } from "../../../../actions/data-source.actions";
import { useState, useEffect } from "react";
import { Spinner, Pagination } from "@heroui/react";
import { ExternalLinkIcon, PlusIcon } from "lucide-react";
@ -13,7 +13,7 @@ import { Textarea } from "@/components/ui/textarea";
import { Section } from "./section";
function UrlListItem({ file, onDelete }: {
file: WithStringId<z.infer<typeof DataSourceDoc>>,
file: z.infer<typeof DataSourceDoc>,
onDelete: (fileId: string) => Promise<void>;
}) {
const [isDeleting, setIsDeleting] = useState(false);
@ -37,7 +37,7 @@ function UrlListItem({ file, onDelete }: {
onClick={async () => {
setIsDeleting(true);
try {
await onDelete(file._id);
await onDelete(file.id);
} finally {
setIsDeleting(false);
}
@ -51,12 +51,11 @@ function UrlListItem({ file, onDelete }: {
);
}
function UrlList({ projectId, sourceId, onDelete }: {
projectId: string,
function UrlList({ sourceId, onDelete }: {
sourceId: string,
onDelete: (fileId: string) => Promise<void>,
}) {
const [files, setFiles] = useState<WithStringId<z.infer<typeof DataSourceDoc>>[]>([]);
const [files, setFiles] = useState<z.infer<typeof DataSourceDoc>[]>([]);
const [loading, setLoading] = useState(true);
const [page, setPage] = useState(1);
const [total, setTotal] = useState(0);
@ -69,7 +68,7 @@ function UrlList({ projectId, sourceId, onDelete }: {
async function fetchFiles() {
setLoading(true);
try {
const { files, total } = await listDocsInDataSource({ projectId, sourceId, page, limit: 10 });
const { files, total } = await listDocsInDataSource({ sourceId, page, limit: 10 });
if (!ignore) {
setFiles(files);
setTotal(total);
@ -86,7 +85,7 @@ function UrlList({ projectId, sourceId, onDelete }: {
return () => {
ignore = true;
};
}, [projectId, sourceId, page]);
}, [sourceId, page]);
return (
<div className="mt-6 space-y-4">
@ -102,7 +101,7 @@ function UrlList({ projectId, sourceId, onDelete }: {
) : (
<div className="space-y-2">
{files.map(file => (
<UrlListItem key={file._id} file={file} onDelete={onDelete} />
<UrlListItem key={file.id} file={file} onDelete={onDelete} />
))}
{Math.ceil(total / 10) > 1 && (
<div className="mt-4">
@ -120,12 +119,10 @@ function UrlList({ projectId, sourceId, onDelete }: {
}
export function ScrapeSource({
projectId,
dataSource,
handleReload,
}: {
projectId: string,
dataSource: WithStringId<z.infer<typeof DataSource>>,
dataSource: z.infer<typeof DataSource>,
handleReload: () => void;
}) {
const [fileListKey, setFileListKey] = useState(0);
@ -161,8 +158,7 @@ export function ScrapeSource({
const first100Urls = urlsArray.slice(0, 100);
await addDocsToDataSource({
projectId,
sourceId: dataSource._id,
sourceId: dataSource.id,
docData: first100Urls.map(url => ({
name: url,
data: {
@ -209,13 +205,10 @@ export function ScrapeSource({
<UrlList
key={fileListKey}
projectId={projectId}
sourceId={dataSource._id}
sourceId={dataSource.id}
onDelete={async (docId) => {
await deleteDocsFromDataSource({
projectId,
sourceId: dataSource._id,
docIds: [docId],
await deleteDocFromDataSource({
docId: docId,
});
handleReload();
setFileListKey(prev => prev + 1);
@ -230,10 +223,8 @@ export function ScrapeSource({
description="Update the content by scraping the URLs again."
>
<Recrawl
projectId={projectId}
sourceId={dataSource._id}
handleRefresh={async () => {
await recrawlWebDataSource(projectId, dataSource._id);
await recrawlWebDataSource(dataSource.id);
handleReload();
setFileListKey(prev => prev + 1);
}}

10
apps/rowboat/app/projects/[projectId]/sources/components/self-updating-source-status.tsx
View file

@ -1,17 +1,15 @@
'use client';
import { getDataSource } from "../../../../actions/data-source.actions";
import { DataSource } from "../../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { useEffect, useState } from "react";
import { z } from 'zod';
import { SourceStatus } from "./source-status";
export function SelfUpdatingSourceStatus({
projectId,
sourceId,
initialStatus,
compact = false,
}: {
projectId: string;
sourceId: string,
initialStatus: z.infer<typeof DataSource>['status'],
compact?: boolean;
@ -26,7 +24,7 @@ export function SelfUpdatingSourceStatus({
if (ignore) {
return;
}
const source = await getDataSource(projectId, sourceId);
const source = await getDataSource(sourceId);
setStatus(source.status);
timeoutId = setTimeout(check, 15 * 1000);
}
@ -41,7 +39,7 @@ export function SelfUpdatingSourceStatus({
clearTimeout(timeoutId);
}
};
}, [status, projectId, sourceId]);
}, [status, sourceId]);
return <SourceStatus status={status} compact={compact} projectId={projectId} />;
return <SourceStatus status={status} compact={compact} />;
}

4
apps/rowboat/app/projects/[projectId]/sources/components/source-status.tsx
View file

@ -1,15 +1,13 @@
import { DataSource } from "../../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { Spinner } from "@heroui/react";
import { z } from 'zod';
import { CheckCircleIcon, XCircleIcon, ClockIcon } from "lucide-react";
export function SourceStatus({
status,
projectId,
compact = false,
}: {
status: z.infer<typeof DataSource>['status'],
projectId: string,
compact?: boolean;
}) {
return (

15
apps/rowboat/app/projects/[projectId]/sources/components/sources-list.tsx
View file

@ -6,15 +6,14 @@ import { ToggleSource } from "./toggle-source";
import { SelfUpdatingSourceStatus } from "./self-updating-source-status";
import { DataSourceIcon } from "../../../../lib/components/datasource-icon";
import { useEffect, useState } from "react";
import { WithStringId } from "../../../../lib/types/types";
import { DataSource } from "../../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { listDataSources } from "../../../../actions/data-source.actions";
import { Panel } from "@/components/common/panel-common";
import { PlusIcon } from "lucide-react";
export function SourcesList({ projectId }: { projectId: string }) {
const [sources, setSources] = useState<WithStringId<z.infer<typeof DataSource>>[]>([]);
const [sources, setSources] = useState<z.infer<typeof DataSource>[]>([]);
const [loading, setLoading] = useState(true);
useEffect(() => {
@ -115,12 +114,12 @@ export function SourcesList({ projectId }: { projectId: string }) {
<tbody className="bg-white dark:bg-gray-800 divide-y divide-gray-200 dark:divide-gray-700">
{sources.map((source) => (
<tr
key={source._id}
key={source.id}
className="hover:bg-gray-50 dark:hover:bg-gray-750 transition-colors"
>
<td className="px-6 py-4 text-left">
<Link
href={`/projects/${projectId}/sources/${source._id}`}
href={`/projects/${projectId}/sources/${source.id}`}
size="lg"
isBlock
className="text-sm text-gray-900 dark:text-gray-100 hover:text-blue-600 dark:hover:text-blue-400 truncate block"
@ -158,8 +157,7 @@ export function SourcesList({ projectId }: { projectId: string }) {
<td className="px-6 py-4 text-left">
<div className="text-sm">
<SelfUpdatingSourceStatus
sourceId={source._id}
projectId={projectId}
sourceId={source.id}
initialStatus={source.status}
compact={true}
/>
@ -168,8 +166,7 @@ export function SourcesList({ projectId }: { projectId: string }) {
)}
<td className="px-6 py-4 text-left">
<ToggleSource
projectId={projectId}
sourceId={source._id}
sourceId={source.id}
active={source.active}
compact={true}
className="bg-default-100"

25
apps/rowboat/app/projects/[projectId]/sources/components/text-source.tsx
View file

@ -1,21 +1,18 @@
"use client";
import { WithStringId } from "../../../../lib/types/types";
import { DataSource } from "../../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { z } from "zod";
import { useState, useEffect } from "react";
import { Textarea } from "@/components/ui/textarea";
import { FormStatusButton } from "../../../../lib/components/form-status-button";
import { Spinner } from "@heroui/react";
import { addDocsToDataSource, deleteDocsFromDataSource, listDocsInDataSource } from "../../../../actions/data-source.actions";
import { addDocsToDataSource, deleteDocFromDataSource, listDocsInDataSource } from "../../../../actions/data-source.actions";
import { Section } from "./section";
export function TextSource({
projectId,
dataSource,
handleReload,
}: {
projectId: string,
dataSource: WithStringId<z.infer<typeof DataSource>>,
dataSource: z.infer<typeof DataSource>,
handleReload: () => void;
}) {
const [content, setContent] = useState("");
@ -30,8 +27,7 @@ export function TextSource({
setIsLoading(true);
try {
const { files } = await listDocsInDataSource({
projectId,
sourceId: dataSource._id,
sourceId: dataSource.id,
limit: 1,
});
@ -41,7 +37,7 @@ export function TextSource({
const doc = files[0];
if (doc.data.type === 'text') {
setContent(doc.data.content);
setDocId(doc._id);
setDocId(doc.id);
}
}
} catch (error) {
@ -55,7 +51,7 @@ export function TextSource({
return () => {
ignore = true;
};
}, [projectId, dataSource._id]);
}, [dataSource.id]);
async function handleSubmit(formData: FormData) {
setIsSaving(true);
@ -64,17 +60,14 @@ export function TextSource({
// Delete existing doc if it exists
if (docId) {
await deleteDocsFromDataSource({
projectId,
sourceId: dataSource._id,
docIds: [docId],
await deleteDocFromDataSource({
docId: docId,
});
}
// Add new doc
await addDocsToDataSource({
projectId,
sourceId: dataSource._id,
sourceId: dataSource.id,
docData: [{
name: 'text',
data: {

4
apps/rowboat/app/projects/[projectId]/sources/components/toggle-source.tsx
View file

@ -4,13 +4,11 @@ import { Spinner } from "@heroui/react";
import { useState } from "react";
export function ToggleSource({
projectId,
sourceId,
active,
compact = false,
className
}: {
projectId: string;
sourceId: string;
active: boolean;
compact?: boolean;
@ -22,7 +20,7 @@ export function ToggleSource({
async function handleToggle() {
setLoading(true);
try {
await toggleDataSource(projectId, sourceId, !isActive);
await toggleDataSource(sourceId, !isActive);
setIsActive(!isActive);
} finally {
setLoading(false);

4
apps/rowboat/app/projects/[projectId]/sources/components/web-recrawl.tsx
View file

@ -3,12 +3,8 @@ import { FormStatusButton } from "../../../../lib/components/form-status-button"
import { RefreshCwIcon } from "lucide-react";
export function Recrawl({
projectId,
sourceId,
handleRefresh,
}: {
projectId: string;
sourceId: string;
handleRefresh: () => void;
}) {
return <form action={handleRefresh}>

12
apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
View file

@ -71,8 +71,7 @@ export function Form({
// pick first 100
const first100Urls = urlsArray.slice(0, 100);
await addDocsToDataSource({
projectId,
sourceId: source._id,
sourceId: source.id,
docData: first100Urls.map(url => ({
name: url,
data: {
@ -82,7 +81,7 @@ export function Form({
})),
});
if (onSuccess) {
onSuccess(source._id);
onSuccess(source.id);
}
}
@ -97,7 +96,7 @@ export function Form({
});
if (onSuccess) {
onSuccess(source._id);
onSuccess(source.id);
}
}
@ -114,8 +113,7 @@ export function Form({
const content = formData.get('content') as string;
await addDocsToDataSource({
projectId,
sourceId: source._id,
sourceId: source.id,
docData: [{
name: 'text',
data: {
@ -126,7 +124,7 @@ export function Form({
});
if (onSuccess) {
onSuccess(source._id);
onSuccess(source.id);
}
}

4
apps/rowboat/app/projects/[projectId]/workflow/app.tsx
View file

@ -1,6 +1,6 @@
"use client";
import { MCPServer, WithStringId } from "../../../lib/types/types";
import { DataSource } from "../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { Project } from "../../../lib/types/project_types";
import { z } from "zod";
import { useCallback, useEffect, useState } from "react";
@ -32,7 +32,7 @@ export function App({
}) {
const [mode, setMode] = useState<'draft' | 'live'>('draft');
const [project, setProject] = useState<WithStringId<z.infer<typeof Project>> | null>(null);
const [dataSources, setDataSources] = useState<WithStringId<z.infer<typeof DataSource>>[] | null>(null);
const [dataSources, setDataSources] = useState<z.infer<typeof DataSource>[] | null>(null);
const [projectConfig, setProjectConfig] = useState<z.infer<typeof Project> | null>(null);
const [loading, setLoading] = useState(false);
const [eligibleModels, setEligibleModels] = useState<z.infer<typeof ModelsResponse> | "*">("*");

8
apps/rowboat/app/projects/[projectId]/workflow/components/DataSourcesModal.tsx
View file

@ -6,8 +6,7 @@ import { Button } from '@/components/ui/button';
import { Form } from '../../sources/new/form';
import { FilesSource } from '../../sources/components/files-source';
import { getDataSource } from '../../../../actions/data-source.actions';
import { WithStringId } from '../../../../lib/types/types';
import { DataSource } from '../../../../lib/types/datasource_types';
import { DataSource } from "@/src/entities/models/data-source";
import { z } from 'zod';
interface DataSourcesModalProps {
@ -30,11 +29,11 @@ export function DataSourcesModal({
useRagScraping
}: DataSourcesModalProps) {
const [currentView, setCurrentView] = useState<'form' | 'upload'>('form');
const [createdSource, setCreatedSource] = useState<WithStringId<z.infer<typeof DataSource>> | null>(null);
const [createdSource, setCreatedSource] = useState<z.infer<typeof DataSource> | null>(null);
const handleDataSourceCreated = async (sourceId: string) => {
// Get the created data source
const source = await getDataSource(projectId, sourceId);
const source = await getDataSource(sourceId);
// If it's a files data source, show the upload interface
if (source.data.type === 'files_local' || source.data.type === 'files_s3') {
@ -93,7 +92,6 @@ export function DataSourcesModal({
) : (
createdSource && (
<FilesSource
projectId={projectId}
dataSource={createdSource}
handleReload={handleFilesUploaded}
type={createdSource.data.type as 'files_local' | 'files_s3'}

14
apps/rowboat/app/projects/[projectId]/workflow/entity_list.tsx
View file

@ -2,7 +2,7 @@ import React, { forwardRef, useImperativeHandle } from "react";
import { z } from "zod";
import { WorkflowPrompt, WorkflowAgent, WorkflowTool, WorkflowPipeline, Workflow } from "../../../lib/types/workflow_types";
import { Project } from "../../../lib/types/project_types";
import { DataSource } from "../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { WithStringId } from "../../../lib/types/types";
import { Dropdown, DropdownItem, DropdownTrigger, DropdownMenu } from "@heroui/react";
import { useRef, useEffect, useState } from "react";
@ -48,7 +48,7 @@ interface EntityListProps {
tools: z.infer<typeof WorkflowTool>[];
prompts: z.infer<typeof WorkflowPrompt>[];
pipelines: z.infer<typeof WorkflowPipeline>[];
dataSources: WithStringId<z.infer<typeof DataSource>>[];
dataSources: z.infer<typeof DataSource>[];
workflow: z.infer<typeof Workflow>;
selectedEntity: {
type: "agent" | "tool" | "prompt" | "datasource" | "pipeline" | "visualise";
@ -1071,14 +1071,14 @@ export const EntityList = forwardRef<
className={clsx(
"flex items-center gap-2 px-3 py-2 rounded-md min-h-[24px] cursor-pointer",
{
"bg-indigo-50 dark:bg-indigo-950/30": selectedEntity?.type === "datasource" && selectedEntity.name === dataSource._id,
"hover:bg-zinc-50 dark:hover:bg-zinc-800": !(selectedEntity?.type === "datasource" && selectedEntity.name === dataSource._id)
"bg-indigo-50 dark:bg-indigo-950/30": selectedEntity?.type === "datasource" && selectedEntity.name === dataSource.id,
"hover:bg-zinc-50 dark:hover:bg-zinc-800": !(selectedEntity?.type === "datasource" && selectedEntity.name === dataSource.id)
}
)}
onClick={() => handleSelectDataSource(dataSource._id)}
onClick={() => handleSelectDataSource(dataSource.id)}
>
<div
ref={selectedEntity?.type === "datasource" && selectedEntity.name === dataSource._id ? selectedRef : undefined}
ref={selectedEntity?.type === "datasource" && selectedEntity.name === dataSource.id ? selectedRef : undefined}
className="flex-1 flex items-center gap-2 text-sm text-left"
>
<div className="shrink-0 flex items-center justify-center w-3 h-3">
@ -1097,7 +1097,7 @@ export const EntityList = forwardRef<
name={dataSource.name}
onDelete={async () => {
if (window.confirm(`Are you sure you want to delete the data source "${dataSource.name}"?`)) {
await deleteDataSource(projectId, dataSource._id);
await deleteDataSource(dataSource.id);
onDataSourcesUpdated?.();
}
}}

4
apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
View file

@ -2,7 +2,7 @@
import React, { useReducer, Reducer, useState, useCallback, useEffect, useRef, createContext, useContext } from "react";
import { MCPServer, Message, WithStringId } from "../../../lib/types/types";
import { Workflow, WorkflowTool, WorkflowPrompt, WorkflowAgent, WorkflowPipeline } from "../../../lib/types/workflow_types";
import { DataSource } from "../../../lib/types/datasource_types";
import { DataSource } from "@/src/entities/models/data-source";
import { Project } from "../../../lib/types/project_types";
import { produce, applyPatches, enablePatches, produceWithPatches, Patch } from 'immer';
import { AgentConfig } from "../entities/agent_config";
@ -821,7 +821,7 @@ export function WorkflowEditor({
chatWidgetHost,
}: {
projectId: string;
dataSources: WithStringId<z.infer<typeof DataSource>>[];
dataSources: z.infer<typeof DataSource>[];
workflow: z.infer<typeof Workflow>;
useRag: boolean;
useRagUploads: boolean;