enable scrape data source rag

apps/rowboat/app/actions.ts

@@ -435,8 +435,18 @@ export async function createCrawlDataSource(projectId: string, formData: FormDat
 export async function createUrlsDataSource(projectId: string, formData: FormData) {
     await projectAuthCheck(projectId);
     const urls = formData.get('urls') as string;
-    // take first 100 urls
+    // take first 100 valid urls (as in parse them)
-    const limitedUrls = urls.split('\n').slice(0, 100).map((url) => url.trim());
+    const limitedUrls = urls.split('\n')
+        .map((url) => url.trim())
+        .filter((url) => {
+            try {
+                new URL(url);
+                return true;
+            } catch (e) {
+                return false;
+            }
+        })
+        .slice(0, 100);
     const name = formData.get('name') as string;
 
     const result = await dataSourcesCollection.insertOne({

apps/rowboat/app/projects/[projectId]/layout.tsx

@@ -7,8 +7,10 @@ export default async function Layout({
     params: { projectId: string }
     children: React.ReactNode
 }) {
+    const useDataSources = process.env.USE_DATA_SOURCES === 'true';
+
     return <div className="flex h-full">
-        <Nav projectId={params.projectId} />
+        <Nav projectId={params.projectId} useDataSources={useDataSources} />
         <div className="grow p-2 overflow-auto bg-white rounded-tl-lg">
             {children}
         </div>

apps/rowboat/app/projects/[projectId]/menu.tsx

@@ -3,7 +3,7 @@ import { usePathname } from "next/navigation";
 import { Tooltip } from "@nextui-org/react";
 import Link from "next/link";
 import clsx from "clsx";
-import { SettingsIcon, WorkflowIcon } from "lucide-react";
+import { DatabaseIcon, SettingsIcon, WorkflowIcon } from "lucide-react";
 
 function NavLink({ href, label, icon, collapsed, selected = false }: { href: string, label: string, icon: React.ReactNode, collapsed: boolean, selected?: boolean }) {
     return <Link
@@ -30,9 +30,11 @@ function NavLink({ href, label, icon, collapsed, selected = false }: { href: str
 export default function Menu({
     projectId,
     collapsed,
+    useDataSources,
 }: {
     projectId: string;
     collapsed: boolean;
+    useDataSources: boolean;
 }) {
     const pathname = usePathname();
 
@@ -53,15 +55,13 @@ export default function Menu({
             icon={<WorkflowIcon size={16} />}
             selected={pathname.startsWith(`/projects/${projectId}/workflow`)}
         />
-        {/* <NavLink
+        {useDataSources && <NavLink
             href={`/projects/${projectId}/sources`}
             label="Data sources"
             collapsed={collapsed}
-            icon=<svg className="w-[24px] h-[24px]" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24">
+            icon={<DatabaseIcon size={16} />}
-                <path stroke="currentColor" strokeLinecap="round" strokeLinejoin="round" strokeWidth="1" d="M19 6c0 1.657-3.134 3-7 3S5 7.657 5 6m14 0c0-1.657-3.134-3-7-3S5 4.343 5 6m14 0v6M5 6v6m0 0c0 1.657 3.134 3 7 3s7-1.343 7-3M5 12v6c0 1.657 3.134 3 7 3s7-1.343 7-3v-6" />
-            </svg>
             selected={pathname.startsWith(`/projects/${projectId}/sources`)}
-        /> */}
+        />}
         <NavLink
             href={`/projects/${projectId}/config`}
             label="Config"

apps/rowboat/app/projects/[projectId]/nav.tsx

@@ -9,8 +9,10 @@ import { ChevronsLeftIcon, ChevronsRightIcon, FolderOpenIcon, PanelLeftCloseIcon
 
 export function Nav({
     projectId,
+    useDataSources,
 }: {
     projectId: string;
+    useDataSources: boolean;
 }) {
     const [collapsed, setCollapsed] = useState(false);
     const [projectName, setProjectName] = useState<string | null>(null);
@@ -57,6 +59,6 @@ export function Nav({
                 <FolderOpenIcon size={16} className="ml-1" />
             </Link>
         </Tooltip>}
-        <Menu projectId={projectId} collapsed={collapsed} />
+        <Menu projectId={projectId} collapsed={collapsed} useDataSources={useDataSources} />
     </div>;
 }

apps/rowboat/app/projects/[projectId]/sources/new/form.tsx

@@ -10,9 +10,9 @@ export function Form({
 }: {
     projectId: string;
 }) {
-    const [sourceType, setSourceType] = useState("crawl");
+    const [sourceType, setSourceType] = useState("");
 
-    const createCrawlDataSourceWithProjectId = createCrawlDataSource.bind(null, projectId);
+    // const createCrawlDataSourceWithProjectId = createCrawlDataSource.bind(null, projectId);
     const createUrlsDataSourceWithProjectId = createUrlsDataSource.bind(null, projectId);
 
     function handleSourceTypeChange(event: React.ChangeEvent<HTMLSelectElement>) {
@@ -26,22 +26,23 @@ export function Form({
                 selectedKeys={[sourceType]}
                 onChange={handleSourceTypeChange}
             >
-                <SelectItem
+                {/* <SelectItem
                     key="crawl"
                     value="crawl"
                     startContent={<DataSourceIcon type="crawl" />}
                 >
                     Crawl URLs
-                </SelectItem>
+                </SelectItem> */}
                 <SelectItem
                     key="urls"
                     value="urls"
                     startContent={<DataSourceIcon type="urls" />}
                 >
-                    Specify URLs
+                    Scrape URLs
                 </SelectItem>
             </Select>
-            {sourceType === "crawl" && <form
+
+            {/* {sourceType === "crawl" && <form
                 action={createCrawlDataSourceWithProjectId}
                 className="flex flex-col gap-4"
             >
@@ -95,7 +96,7 @@ export function Form({
                         </svg>,
                     }}
                 />
-            </form>}
+            </form>} */}
 
             {sourceType === "urls" && <form
                 action={createUrlsDataSourceWithProjectId}
@@ -127,7 +128,7 @@ export function Form({
                     <p>Note:</p>
                     <ul className="list-disc ml-4">
                         <li>Expect about 5-10 minutes to scrape 100 pages</li>
-                        <li>Only the first 100 URLs will be scraped</li>
+                        <li>Only the first 100 (valid) URLs will be scraped</li>
                     </ul>
                 </div>
                 <FormStatusButton

apps/rowboat/app/projects/[projectId]/sources/new/page.tsx

@@ -1,5 +1,6 @@
 import { Metadata } from "next";
 import { Form } from "./form";
+import { redirect } from "next/navigation";
 
 export const metadata: Metadata = {
     title: "Add data source"
@@ -10,6 +11,12 @@ export default async function Page({
 }: {
     params: { projectId: string }
 }) {
+    const useDataSources = process.env.USE_DATA_SOURCES === 'true';
+
+    if (!useDataSources) {
+        redirect(`/projects/${params.projectId}`);
+    }
+
     return <div className="flex flex-col h-full">
         <div className="shrink-0 flex justify-between items-center pb-4 border-b border-b-gray-100">
             <div className="flex flex-col">