From 58d3c5eebbe5783475150dc0697a57522a1f8f08 Mon Sep 17 00:00:00 2001
From: ramnique <30795890+ramnique@users.noreply.github.com>
Date: Wed, 5 Feb 2025 16:49:48 +0530
Subject: [PATCH 1/3] autosave on undo/redo

---
 .../app/projects/[projectId]/workflow/workflow_editor.tsx   | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx b/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
index db04437b..8ecbf0b8 100644
--- a/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
+++ b/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
@@ -149,6 +149,8 @@ function reducer(state: State, action: Action): State {
                 const inverse = state.inversePatches[state.currentIndex - 1];
                 draft.present = applyPatches(state.present, inverse);
                 draft.currentIndex--;
+                draft.present.pendingChanges = true;
+                draft.present.chatKey++;
             });
             break;
         }
@@ -158,7 +160,9 @@ function reducer(state: State, action: Action): State {
                 const patch = state.patches[state.currentIndex];
                 draft.present = applyPatches(state.present, patch);
                 draft.currentIndex++;
-            });
+                draft.present.pendingChanges = true;
+                draft.present.chatKey++;
+             });
             break;
         }
         case "update_workflow_name": {

From a1adbd722c6b1bba70c79f96b66d4f0cf6bf8bc6 Mon Sep 17 00:00:00 2001
From: ramnique <30795890+ramnique@users.noreply.github.com>
Date: Wed, 5 Feb 2025 18:12:34 +0530
Subject: [PATCH 2/3] use only firecrawl for scraping, dont rewrite

---
 apps/rowboat/app/scripts/crawlUrls.ts | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/apps/rowboat/app/scripts/crawlUrls.ts b/apps/rowboat/app/scripts/crawlUrls.ts
index 62536ef0..02474d77 100644
--- a/apps/rowboat/app/scripts/crawlUrls.ts
+++ b/apps/rowboat/app/scripts/crawlUrls.ts
@@ -685,6 +685,18 @@ async function saveWebpagesToMongodb(logger: PrefixLogger, job: WithId<z.infer<t
     });
 }
 
+async function dummyRewrite(_logger: PrefixLogger) {
+    const logger = _logger.child('dummyRewrite');
+    await batchMode({
+        inputFilePath: 'crawled.jsonl',
+        outputFilePath: 'rewritten.jsonl',
+        batchSize: 10,
+        processBatch: async (batch: Webpage[]): Promise<Webpage[]> => {
+            return batch;
+        }
+    });
+}
+
 async function rewrite(_logger: PrefixLogger) {
     const logger = _logger.child('rewrite');
 
@@ -950,7 +962,8 @@ async function mongodb(logger: PrefixLogger, job: WithId<z.infer<typeof DataSour
                 await dataSourcesCollection.updateOne({ _id: job._id }, { $set: { 'data.scrapedUrls': Array.from(crawledUrls).join('\n') } });
             }
             // rewrite the merged results as simplified html and markdown
-            await rewrite(logger);
+            // await rewrite(logger);
+            await dummyRewrite(logger);
             await saveWebpagesToMongodb(logger, job);
             await chunk(logger, job);
             await embeddings(logger);

From e497e3ce764f845a3ff84936c7f6c6c47d9ac7b0 Mon Sep 17 00:00:00 2001
From: ramnique <30795890+ramnique@users.noreply.github.com>
Date: Wed, 5 Feb 2025 18:50:11 +0530
Subject: [PATCH 3/3] enable scrape data source rag

---
 apps/rowboat/app/actions.ts                     | 14 ++++++++++++--
 .../rowboat/app/projects/[projectId]/layout.tsx |  4 +++-
 apps/rowboat/app/projects/[projectId]/menu.tsx  | 12 ++++++------
 apps/rowboat/app/projects/[projectId]/nav.tsx   |  4 +++-
 .../projects/[projectId]/sources/new/form.tsx   | 17 +++++++++--------
 .../projects/[projectId]/sources/new/page.tsx   |  7 +++++++
 6 files changed, 40 insertions(+), 18 deletions(-)

diff --git a/apps/rowboat/app/actions.ts b/apps/rowboat/app/actions.ts
index fef6f4ab..af06afc0 100644
--- a/apps/rowboat/app/actions.ts
+++ b/apps/rowboat/app/actions.ts
@@ -435,8 +435,18 @@ export async function createCrawlDataSource(projectId: string, formData: FormDat
 export async function createUrlsDataSource(projectId: string, formData: FormData) {
     await projectAuthCheck(projectId);
     const urls = formData.get('urls') as string;
-    // take first 100 urls
-    const limitedUrls = urls.split('\n').slice(0, 100).map((url) => url.trim());
+    // take first 100 valid urls (as in parse them)
+    const limitedUrls = urls.split('\n')
+        .map((url) => url.trim())
+        .filter((url) => {
+            try {
+                new URL(url);
+                return true;
+            } catch (e) {
+                return false;
+            }
+        })
+        .slice(0, 100);
     const name = formData.get('name') as string;
 
     const result = await dataSourcesCollection.insertOne({
diff --git a/apps/rowboat/app/projects/[projectId]/layout.tsx b/apps/rowboat/app/projects/[projectId]/layout.tsx
index 6b0bb0a9..108d21bd 100644
--- a/apps/rowboat/app/projects/[projectId]/layout.tsx
+++ b/apps/rowboat/app/projects/[projectId]/layout.tsx
@@ -7,8 +7,10 @@ export default async function Layout({
     params: { projectId: string }
     children: React.ReactNode
 }) {
+    const useDataSources = process.env.USE_DATA_SOURCES === 'true';
+
     return <div className="flex h-full">
-        <Nav projectId={params.projectId} />
+        <Nav projectId={params.projectId} useDataSources={useDataSources} />
         <div className="grow p-2 overflow-auto bg-white rounded-tl-lg">
             {children}
         </div>
diff --git a/apps/rowboat/app/projects/[projectId]/menu.tsx b/apps/rowboat/app/projects/[projectId]/menu.tsx
index 23f77e78..f5e15944 100644
--- a/apps/rowboat/app/projects/[projectId]/menu.tsx
+++ b/apps/rowboat/app/projects/[projectId]/menu.tsx
@@ -3,7 +3,7 @@ import { usePathname } from "next/navigation";
 import { Tooltip } from "@nextui-org/react";
 import Link from "next/link";
 import clsx from "clsx";
-import { SettingsIcon, WorkflowIcon } from "lucide-react";
+import { DatabaseIcon, SettingsIcon, WorkflowIcon } from "lucide-react";
 
 function NavLink({ href, label, icon, collapsed, selected = false }: { href: string, label: string, icon: React.ReactNode, collapsed: boolean, selected?: boolean }) {
     return <Link
@@ -30,9 +30,11 @@ function NavLink({ href, label, icon, collapsed, selected = false }: { href: str
 export default function Menu({
     projectId,
     collapsed,
+    useDataSources,
 }: {
     projectId: string;
     collapsed: boolean;
+    useDataSources: boolean;
 }) {
     const pathname = usePathname();
 
@@ -53,15 +55,13 @@ export default function Menu({
             icon={<WorkflowIcon size={16} />}
             selected={pathname.startsWith(`/projects/${projectId}/workflow`)}
         />
-        {/* <NavLink
+        {useDataSources && <NavLink
             href={`/projects/${projectId}/sources`}
             label="Data sources"
             collapsed={collapsed}
-            icon=<svg className="w-[24px] h-[24px]" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24">
-                <path stroke="currentColor" strokeLinecap="round" strokeLinejoin="round" strokeWidth="1" d="M19 6c0 1.657-3.134 3-7 3S5 7.657 5 6m14 0c0-1.657-3.134-3-7-3S5 4.343 5 6m14 0v6M5 6v6m0 0c0 1.657 3.134 3 7 3s7-1.343 7-3M5 12v6c0 1.657 3.134 3 7 3s7-1.343 7-3v-6" />
-            </svg>
+            icon={<DatabaseIcon size={16} />}
             selected={pathname.startsWith(`/projects/${projectId}/sources`)}
-        /> */}
+        />}
         <NavLink
             href={`/projects/${projectId}/config`}
             label="Config"
diff --git a/apps/rowboat/app/projects/[projectId]/nav.tsx b/apps/rowboat/app/projects/[projectId]/nav.tsx
index 89fbe0f0..e357c453 100644
--- a/apps/rowboat/app/projects/[projectId]/nav.tsx
+++ b/apps/rowboat/app/projects/[projectId]/nav.tsx
@@ -9,8 +9,10 @@ import { ChevronsLeftIcon, ChevronsRightIcon, FolderOpenIcon, PanelLeftCloseIcon
 
 export function Nav({
     projectId,
+    useDataSources,
 }: {
     projectId: string;
+    useDataSources: boolean;
 }) {
     const [collapsed, setCollapsed] = useState(false);
     const [projectName, setProjectName] = useState<string | null>(null);
@@ -57,6 +59,6 @@ export function Nav({
                 <FolderOpenIcon size={16} className="ml-1" />
             </Link>
         </Tooltip>}
-        <Menu projectId={projectId} collapsed={collapsed} />
+        <Menu projectId={projectId} collapsed={collapsed} useDataSources={useDataSources} />
     </div>;
 }
\ No newline at end of file
diff --git a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
index e9c0ba5f..d47cb9a3 100644
--- a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
+++ b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
@@ -10,9 +10,9 @@ export function Form({
 }: {
     projectId: string;
 }) {
-    const [sourceType, setSourceType] = useState("crawl");
+    const [sourceType, setSourceType] = useState("");
 
-    const createCrawlDataSourceWithProjectId = createCrawlDataSource.bind(null, projectId);
+    // const createCrawlDataSourceWithProjectId = createCrawlDataSource.bind(null, projectId);
     const createUrlsDataSourceWithProjectId = createUrlsDataSource.bind(null, projectId);
 
     function handleSourceTypeChange(event: React.ChangeEvent<HTMLSelectElement>) {
@@ -26,22 +26,23 @@ export function Form({
                 selectedKeys={[sourceType]}
                 onChange={handleSourceTypeChange}
             >
-                <SelectItem
+                {/* <SelectItem
                     key="crawl"
                     value="crawl"
                     startContent={<DataSourceIcon type="crawl" />}
                 >
                     Crawl URLs
-                </SelectItem>
+                </SelectItem> */}
                 <SelectItem
                     key="urls"
                     value="urls"
                     startContent={<DataSourceIcon type="urls" />}
                 >
-                    Specify URLs
+                    Scrape URLs
                 </SelectItem>
             </Select>
-            {sourceType === "crawl" && <form
+
+            {/* {sourceType === "crawl" && <form
                 action={createCrawlDataSourceWithProjectId}
                 className="flex flex-col gap-4"
             >
@@ -95,7 +96,7 @@ export function Form({
                         </svg>,
                     }}
                 />
-            </form>}
+            </form>} */}
 
             {sourceType === "urls" && <form
                 action={createUrlsDataSourceWithProjectId}
@@ -127,7 +128,7 @@ export function Form({
                     <p>Note:</p>
                     <ul className="list-disc ml-4">
                         <li>Expect about 5-10 minutes to scrape 100 pages</li>
-                        <li>Only the first 100 URLs will be scraped</li>
+                        <li>Only the first 100 (valid) URLs will be scraped</li>
                     </ul>
                 </div>
                 <FormStatusButton
diff --git a/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx b/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx
index 34cfe989..f5a6dd9a 100644
--- a/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx
+++ b/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx
@@ -1,5 +1,6 @@
 import { Metadata } from "next";
 import { Form } from "./form";
+import { redirect } from "next/navigation";
 
 export const metadata: Metadata = {
     title: "Add data source"
@@ -10,6 +11,12 @@ export default async function Page({
 }: {
     params: { projectId: string }
 }) {
+    const useDataSources = process.env.USE_DATA_SOURCES === 'true';
+
+    if (!useDataSources) {
+        redirect(`/projects/${params.projectId}`);
+    }
+
     return <div className="flex flex-col h-full">
         <div className="shrink-0 flex justify-between items-center pb-4 border-b border-b-gray-100">
             <div className="flex flex-col">