From cbac042003e5f92d332499e98fb5a19a738d656c Mon Sep 17 00:00:00 2001
From: ramnique <30795890+ramnique@users.noreply.github.com>
Date: Mon, 10 Mar 2025 12:03:06 +0530
Subject: [PATCH] Update feature flags + env docs

---
 .env.example                                  | 28 +++++-
 .../rowboat/app/actions/datasource_actions.ts |  4 +-
 apps/rowboat/app/actions/project_actions.ts   |  1 +
 apps/rowboat/app/lib/feature_flags.ts         |  4 +
 .../app/projects/[projectId]/config/app.tsx   |  4 +-
 .../app/projects/[projectId]/config/page.tsx  |  5 +-
 .../app/projects/[projectId]/layout.tsx       |  5 +-
 .../rowboat/app/projects/[projectId]/menu.tsx |  6 +-
 apps/rowboat/app/projects/[projectId]/nav.tsx |  8 +-
 .../[projectId]/playground/messages.tsx       | 14 +--
 .../projects/[projectId]/sources/new/form.tsx | 87 ++---------------
 .../projects/[projectId]/sources/new/page.tsx | 11 ++-
 .../[projectId]/workflow/agent_config.tsx     |  9 +-
 .../app/projects/[projectId]/workflow/app.tsx |  4 +-
 .../projects/[projectId]/workflow/page.tsx    |  4 +
 .../[projectId]/workflow/workflow_editor.tsx  |  3 +
 apps/rowboat/app/scripts/rag_files_worker.ts  | 10 +-
 apps/rowboat/app/scripts/rag_urls_worker.ts   | 28 +-----
 apps/rowboat/scripts.Dockerfile               | 46 +++++++++
 docker-compose.yml                            | 97 +++++++++++++------
 20 files changed, 210 insertions(+), 168 deletions(-)
 create mode 100644 apps/rowboat/app/lib/feature_flags.ts
 create mode 100644 apps/rowboat/scripts.Dockerfile

diff --git a/.env.example b/.env.example
index 8351ad5b..cce09ad6 100644
--- a/.env.example
+++ b/.env.example
@@ -1,9 +1,31 @@
+# Basic configuration
+# ------------------------------------------------------------
+MONGODB_CONNECTION_STRING=mongodb://127.0.0.1:27017/rowboat
 OPENAI_API_KEY=<OPENAI_API_KEY>
-MONGODB_CONNECTION_STRING=<MONGODB_CONNECTION_STRING>
 AUTH0_SECRET=<AUTH0_SECRET>
 AUTH0_BASE_URL=http://localhost:3000
 AUTH0_ISSUER_BASE_URL=<AUTH0_ISSUER_BASE_URL>
 AUTH0_CLIENT_ID=<AUTH0_CLIENT_ID>
 AUTH0_CLIENT_SECRET=<AUTH0_CLIENT_SECRET>
-COPILOT_API_KEY=test
-AGENTS_API_KEY=test
\ No newline at end of file
+
+# Uncomment to enable RAG:
+# ------------------------------------------------------------
+# USE_RAG=true
+
+# Uncomment to enable RAG: File uploads
+# ------------------------------------------------------------
+# USE_RAG_UPLOADS=true
+# AWS_ACCESS_KEY_ID=<AWS_ACCESS_KEY_ID>
+# AWS_SECRET_ACCESS_KEY=<AWS_SECRET_ACCESS_KEY>
+# RAG_UPLOADS_S3_BUCKET=<RAG_UPLOADS_S3_BUCKET>
+# RAG_UPLOADS_S3_REGION=<RAG_UPLOADS_S3_REGION>
+
+# Uncomment to enable RAG: Scraping URLs
+# ------------------------------------------------------------
+# USE_RAG_SCRAPING=true
+# FIRECRAWL_API_KEY=<FIRECRAWL_API_KEY>
+
+# Uncomment to enable chat widget
+# ------------------------------------------------------------
+# USE_CHAT_WIDGET=true
+# CHAT_WIDGET_SESSION_JWT_SECRET=<CHAT_WIDGET_SESSION_JWT_SECRET>
\ No newline at end of file
diff --git a/apps/rowboat/app/actions/datasource_actions.ts b/apps/rowboat/app/actions/datasource_actions.ts
index b9159c96..806e1709 100644
--- a/apps/rowboat/app/actions/datasource_actions.ts
+++ b/apps/rowboat/app/actions/datasource_actions.ts
@@ -294,7 +294,7 @@ export async function getDownloadUrlForFile(
     }
 
     const command = new GetObjectCommand({
-        Bucket: process.env.UPLOADS_S3_BUCKET,
+        Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
         Key: file.data.s3Key,
     });
 
@@ -328,7 +328,7 @@ export async function getUploadUrlsForFilesDataSource(
         const s3Key = `datasources/files/${projectIdPrefix}/${projectId}/${sourceId}/${fileId}/${file.name}`;
         // Generate presigned URL
         const command = new PutObjectCommand({
-            Bucket: process.env.UPLOADS_S3_BUCKET,
+            Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
             Key: s3Key,
             ContentType: file.type,
         });
diff --git a/apps/rowboat/app/actions/project_actions.ts b/apps/rowboat/app/actions/project_actions.ts
index 163501cf..b7a1762f 100644
--- a/apps/rowboat/app/actions/project_actions.ts
+++ b/apps/rowboat/app/actions/project_actions.ts
@@ -53,6 +53,7 @@ export async function createProject(formData: FormData) {
         secret,
         nextWorkflowNumber: 1,
         testRunCounter: 0,
+        webhookUrl: 'http://tools_webhook:3005/tool_call',
     });
 
     // add first workflow version
diff --git a/apps/rowboat/app/lib/feature_flags.ts b/apps/rowboat/app/lib/feature_flags.ts
new file mode 100644
index 00000000..bb48726e
--- /dev/null
+++ b/apps/rowboat/app/lib/feature_flags.ts
@@ -0,0 +1,4 @@
+export const USE_RAG = process.env.USE_RAG === 'true';
+export const USE_RAG_UPLOADS = process.env.USE_RAG_UPLOADS === 'true';
+export const USE_RAG_SCRAPING = process.env.USE_RAG_SCRAPING === 'true';
+export const USE_CHAT_WIDGET = process.env.USE_CHAT_WIDGET === 'true';
\ No newline at end of file
diff --git a/apps/rowboat/app/projects/[projectId]/config/app.tsx b/apps/rowboat/app/projects/[projectId]/config/app.tsx
index 4b094658..cb4944c6 100644
--- a/apps/rowboat/app/projects/[projectId]/config/app.tsx
+++ b/apps/rowboat/app/projects/[projectId]/config/app.tsx
@@ -569,9 +569,11 @@ export function DeleteProjectSection({
 
 export default function App({
     projectId,
+    useChatWidget,
     chatWidgetHost,
 }: {
     projectId: string;
+    useChatWidget: boolean;
     chatWidgetHost: string;
 }) {
     return <div className="flex flex-col h-full">
@@ -586,7 +588,7 @@ export default function App({
                 <SecretSection projectId={projectId} />
                 <ApiKeysSection projectId={projectId} />
                 <WebhookUrlSection projectId={projectId} />
-                <ChatWidgetSection projectId={projectId} chatWidgetHost={chatWidgetHost} />
+                {useChatWidget && <ChatWidgetSection projectId={projectId} chatWidgetHost={chatWidgetHost} />}
                 <DeleteProjectSection projectId={projectId} />
             </div>
         </div>
diff --git a/apps/rowboat/app/projects/[projectId]/config/page.tsx b/apps/rowboat/app/projects/[projectId]/config/page.tsx
index e0ee6cfc..a0577fc3 100644
--- a/apps/rowboat/app/projects/[projectId]/config/page.tsx
+++ b/apps/rowboat/app/projects/[projectId]/config/page.tsx
@@ -1,5 +1,7 @@
 import { Metadata } from "next";
 import App from "./app";
+import { USE_CHAT_WIDGET } from "@/app/lib/feature_flags";
+
 export const metadata: Metadata = {
     title: "Project config",
 };
@@ -13,6 +15,7 @@ export default function Page({
 }) {
     return <App
         projectId={params.projectId}
-        chatWidgetHost={process.env.CHAT_WIDGET_HOST || 'https://chat.rowboatlabs.com'}
+        useChatWidget={USE_CHAT_WIDGET}
+        chatWidgetHost={process.env.CHAT_WIDGET_HOST || ''}
     />;
 }
\ No newline at end of file
diff --git a/apps/rowboat/app/projects/[projectId]/layout.tsx b/apps/rowboat/app/projects/[projectId]/layout.tsx
index 21c9fbfb..c871c175 100644
--- a/apps/rowboat/app/projects/[projectId]/layout.tsx
+++ b/apps/rowboat/app/projects/[projectId]/layout.tsx
@@ -1,4 +1,5 @@
 import { Nav } from "./nav";
+import { USE_RAG } from "@/app/lib/feature_flags";
 
 export default async function Layout({
     params,
@@ -7,10 +8,10 @@ export default async function Layout({
     params: { projectId: string }
     children: React.ReactNode
 }) {
-    const useDataSources = process.env.USE_DATA_SOURCES === 'true';
+    const useRag = USE_RAG;
 
     return <div className="flex h-full">
-        <Nav projectId={params.projectId} useDataSources={useDataSources} />
+        <Nav projectId={params.projectId} useRag={useRag} />
         <div className="grow p-2 overflow-auto bg-background dark:bg-background rounded-tl-lg">
             {children}
         </div>
diff --git a/apps/rowboat/app/projects/[projectId]/menu.tsx b/apps/rowboat/app/projects/[projectId]/menu.tsx
index 0a394abc..a1551a4c 100644
--- a/apps/rowboat/app/projects/[projectId]/menu.tsx
+++ b/apps/rowboat/app/projects/[projectId]/menu.tsx
@@ -44,11 +44,11 @@ function NavLink({ href, label, icon, collapsed, selected = false }: {
 export default function Menu({
     projectId,
     collapsed,
-    useDataSources,
+    useRag,
 }: {
     projectId: string;
     collapsed: boolean;
-    useDataSources: boolean;
+    useRag: boolean;
 }) {
     const pathname = usePathname();
 
@@ -68,7 +68,7 @@ export default function Menu({
                 icon={<PlayIcon size={16} />}
                 selected={pathname.startsWith(`/projects/${projectId}/test`)}
             />
-            {useDataSources && (
+            {useRag && (
                 <NavLink
                     href={`/projects/${projectId}/sources`}
                     label="Connect"
diff --git a/apps/rowboat/app/projects/[projectId]/nav.tsx b/apps/rowboat/app/projects/[projectId]/nav.tsx
index b710b645..e941c5b3 100644
--- a/apps/rowboat/app/projects/[projectId]/nav.tsx
+++ b/apps/rowboat/app/projects/[projectId]/nav.tsx
@@ -5,14 +5,14 @@ import { useEffect, useState } from "react";
 import clsx from "clsx";
 import Menu from "./menu";
 import { getProjectConfig } from "../../actions/project_actions";
-import { ChevronsLeftIcon, ChevronsRightIcon, FolderOpenIcon, PanelLeftCloseIcon, PanelLeftOpenIcon } from "lucide-react";
+import { FolderOpenIcon, PanelLeftCloseIcon, PanelLeftOpenIcon } from "lucide-react";
 
 export function Nav({
     projectId,
-    useDataSources,
+    useRag,
 }: {
     projectId: string;
-    useDataSources: boolean;
+    useRag: boolean;
 }) {
     const [collapsed, setCollapsed] = useState(false);
     const [projectName, setProjectName] = useState<string | null>(null);
@@ -56,6 +56,6 @@ export function Nav({
                 <FolderOpenIcon size={16} className="ml-1" />
             </Link>
         </Tooltip>}
-        <Menu projectId={projectId} collapsed={collapsed} useDataSources={useDataSources} />
+        <Menu projectId={projectId} collapsed={collapsed} useRag={useRag} />
     </div>;
 }
\ No newline at end of file
diff --git a/apps/rowboat/app/projects/[projectId]/playground/messages.tsx b/apps/rowboat/app/projects/[projectId]/playground/messages.tsx
index 7c2418cc..5b3c6045 100644
--- a/apps/rowboat/app/projects/[projectId]/playground/messages.tsx
+++ b/apps/rowboat/app/projects/[projectId]/playground/messages.tsx
@@ -186,26 +186,28 @@ function ToolCall({
                     sender={sender}
                 />;
             }
-            if (matchingWorkflowTool && testProfile && !testProfile.mockTools) {
-                return <ClientToolCall
+            if (!matchingWorkflowTool || 
+                matchingWorkflowTool.mockTool || 
+                (testProfile && testProfile.mockTools)) {
+                return <MockToolCall
                     toolCall={toolCall}
                     result={result}
                     handleResult={handleResult}
                     projectId={projectId}
                     messages={messages}
                     sender={sender}
+                    testProfile={testProfile}
+                    workflowTool={matchingWorkflowTool}
+                    systemMessage={systemMessage}
                 />;
             }
-            return <MockToolCall
+            return <ClientToolCall
                 toolCall={toolCall}
                 result={result}
                 handleResult={handleResult}
                 projectId={projectId}
                 messages={messages}
                 sender={sender}
-                testProfile={testProfile}
-                workflowTool={matchingWorkflowTool}
-                systemMessage={systemMessage}
             />;
     }
 }
diff --git a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
index 39601e32..0c12e83b 100644
--- a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
+++ b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx
@@ -8,27 +8,17 @@ import { PlusIcon } from "lucide-react";
 import { useRouter } from "next/navigation";
 
 export function Form({
-    projectId
+    projectId,
+    useRagUploads,
+    useRagScraping,
 }: {
     projectId: string;
+    useRagUploads: boolean;
+    useRagScraping: boolean;
 }) {
     const [sourceType, setSourceType] = useState("");
     const router = useRouter();
 
-    // async function createCrawlDataSource(formData: FormData) {
-    //     const source = await createDataSource({
-    //         projectId,
-    //         name: formData.get('name') as string,
-    //         data: {
-    //             type: 'crawl',
-    //             startUrl: formData.get('startUrl') as string,
-    //             limit: parseInt(formData.get('limit') as string),
-    //         },
-    //         status: 'queued',
-    //     });
-    //     router.push(`/projects/${projectId}/sources/${source._id}`);
-    // }
-
     async function createUrlsDataSource(formData: FormData) {
         const source = await createDataSource({
             projectId,
@@ -80,14 +70,11 @@ export function Form({
                 label="Select type"
                 selectedKeys={[sourceType]}
                 onChange={handleSourceTypeChange}
+                disabledKeys={[
+                    ...(useRagUploads ? [] : ['files']),
+                    ...(useRagScraping ? [] : ['urls']),
+                ]}
             >
-                {/* <SelectItem
-                    key="crawl"
-                    value="crawl"
-                    startContent={<DataSourceIcon type="crawl" />}
-                >
-                    Crawl URLs
-                </SelectItem> */}
                 <SelectItem
                     key="urls"
                     startContent={<DataSourceIcon type="urls" />}
@@ -102,62 +89,6 @@ export function Form({
                 </SelectItem>
              </Select>
 
-            {/* {sourceType === "crawl" && <form
-                action={createCrawlDataSourceWithProjectId}
-                className="flex flex-col gap-4"
-            >
-                <Input
-                    required
-                    type="text"
-                    name="url"
-                    label="Specify starting URL to crawl"
-                    labelPlacement="outside"
-                    placeholder="https://example.com"
-                    variant="bordered"
-                />
-                <div className="self-start w-[200px]">
-                    <Input
-                        required
-                        type="number"
-                        min={1}
-                        max={5000}
-                        name="limit"
-                        label="Maximum pages to crawl"
-                        labelPlacement="outside"
-                        placeholder="100"
-                        defaultValue={"100"}
-                        variant="bordered"
-                    />
-                </div>
-                <div className="self-start">
-                    <Input
-                        required
-                        type="text"
-                        name="name"
-                        label="Name this data source"
-                        labelPlacement="outside"
-                        placeholder="e.g. Help articles"
-                        variant="bordered"
-                    />
-                </div>
-                <div className="text-sm">
-                    <p>Note:</p>
-                    <ul className="list-disc ml-4">
-                        <li>Expect about 5-10 minutes to crawl 100 pages</li>
-                    </ul>
-                </div>
-                <FormStatusButton
-                    props={{
-                        type: "submit",
-                        children: "Add data source",
-                        className: "self-start",
-                        startContent: <svg className="w-[24px] h-[24px]" aria-hidden="true" xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24">
-                            <path stroke="currentColor" strokeLinecap="round" strokeLinejoin="round" strokeWidth="1" d="M5 12h14m-7 7V5" />
-                        </svg>,
-                    }}
-                />
-            </form>} */}
-
             {sourceType === "urls" && <form
                 action={createUrlsDataSource}
                 className="flex flex-col gap-4"
diff --git a/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx b/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx
index 3df5ec5b..4355cb46 100644
--- a/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx
+++ b/apps/rowboat/app/projects/[projectId]/sources/new/page.tsx
@@ -1,6 +1,7 @@
 import { Metadata } from "next";
 import { Form } from "./form";
 import { redirect } from "next/navigation";
+import { USE_RAG, USE_RAG_UPLOADS, USE_RAG_SCRAPING } from "../../../../lib/feature_flags";
 
 export const metadata: Metadata = {
     title: "Add data source"
@@ -11,9 +12,7 @@ export default async function Page({
 }: {
     params: { projectId: string }
 }) {
-    const useDataSources = process.env.USE_DATA_SOURCES === 'true';
-
-    if (!useDataSources) {
+    if (!USE_RAG) {
         redirect(`/projects/${params.projectId}`);
     }
 
@@ -23,6 +22,10 @@ export default async function Page({
                 <h1 className="text-lg">Add data source</h1>
             </div>
         </div>
-        <Form projectId={params.projectId} />
+        <Form
+            projectId={params.projectId}
+            useRagUploads={USE_RAG_UPLOADS}
+            useRagScraping={USE_RAG_SCRAPING}
+        />
     </div>;
 }
\ No newline at end of file
diff --git a/apps/rowboat/app/projects/[projectId]/workflow/agent_config.tsx b/apps/rowboat/app/projects/[projectId]/workflow/agent_config.tsx
index d250bfae..393ac83c 100644
--- a/apps/rowboat/app/projects/[projectId]/workflow/agent_config.tsx
+++ b/apps/rowboat/app/projects/[projectId]/workflow/agent_config.tsx
@@ -10,8 +10,7 @@ import { ActionButton, StructuredPanel } from "../../../lib/components/structure
 import { FormSection } from "../../../lib/components/form-section";
 import { EditableField } from "../../../lib/components/editable-field";
 import { Label } from "../../../lib/components/label";
-import { PlusIcon, SparklesIcon, ChevronRight, ChevronDown } from "lucide-react";
-import { List } from "./config_list";
+import { PlusIcon, ChevronRight, ChevronDown } from "lucide-react";
 import { useState, useEffect, useRef } from "react";
 import { usePreviewModal } from "./preview-modal";
 import { Modal, ModalContent, ModalHeader, ModalBody, ModalFooter } from "@heroui/react";
@@ -33,6 +32,7 @@ export function AgentConfig({
     dataSources,
     handleUpdate,
     handleClose,
+    useRag,
 }: {
     projectId: string,
     workflow: z.infer<typeof Workflow>,
@@ -44,6 +44,7 @@ export function AgentConfig({
     dataSources: WithStringId<z.infer<typeof DataSource>>[],
     handleUpdate: (agent: z.infer<typeof WorkflowAgent>) => void,
     handleClose: () => void,
+    useRag: boolean,
 }) {
     const [isAdvancedConfigOpen, setIsAdvancedConfigOpen] = useState(false);
 
@@ -160,7 +161,7 @@ export function AgentConfig({
                     />
                 </FormSection>
 
-                <FormSection label="RAG (beta)" showDivider>
+                {useRag && <FormSection label="RAG (beta)" showDivider>
                     <div className="flex flex-col gap-3">
                         <Dropdown>
                             <DropdownTrigger>
@@ -272,7 +273,7 @@ export function AgentConfig({
                             </>
                         )}
                     </div>
-                </FormSection>
+                </FormSection>}
 
                 <FormSection label="Model" showDivider>
                     <CustomDropdown
diff --git a/apps/rowboat/app/projects/[projectId]/workflow/app.tsx b/apps/rowboat/app/projects/[projectId]/workflow/app.tsx
index df8a18cb..47dae09e 100644
--- a/apps/rowboat/app/projects/[projectId]/workflow/app.tsx
+++ b/apps/rowboat/app/projects/[projectId]/workflow/app.tsx
@@ -9,12 +9,13 @@ import { WorkflowSelector } from "./workflow_selector";
 import { Spinner } from "@heroui/react";
 import { cloneWorkflow, createWorkflow, fetchPublishedWorkflowId, fetchWorkflow } from "../../../actions/workflow_actions";
 import { listDataSources } from "../../../actions/datasource_actions";
-import { TestProfile } from "@/app/lib/types/testing_types";
 
 export function App({
     projectId,
+    useRag,
 }: {
     projectId: string;
+    useRag: boolean;
 }) {
     const [selectorKey, setSelectorKey] = useState(0);
     const [workflow, setWorkflow] = useState<WithStringId<z.infer<typeof Workflow>> | null>(null);
@@ -106,6 +107,7 @@ export function App({
             publishedWorkflowId={publishedWorkflowId}
             handleShowSelector={handleShowSelector}
             handleCloneVersion={handleCloneVersion}
+            useRag={useRag}
         />}
     </>
 }
diff --git a/apps/rowboat/app/projects/[projectId]/workflow/page.tsx b/apps/rowboat/app/projects/[projectId]/workflow/page.tsx
index 72eb7240..e4d99289 100644
--- a/apps/rowboat/app/projects/[projectId]/workflow/page.tsx
+++ b/apps/rowboat/app/projects/[projectId]/workflow/page.tsx
@@ -1,5 +1,8 @@
 import { Metadata } from "next";
 import { App } from "./app";
+import { USE_RAG } from "@/app/lib/feature_flags";
+
+export const dynamic = 'force-dynamic';
 
 export const metadata: Metadata = {
     title: "Workflow"
@@ -12,5 +15,6 @@ export default async function Page({
 }) {
     return <App
         projectId={params.projectId}
+        useRag={USE_RAG}
     />;
 }
diff --git a/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx b/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
index d97d7a84..4923e067 100644
--- a/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
+++ b/apps/rowboat/app/projects/[projectId]/workflow/workflow_editor.tsx
@@ -534,12 +534,14 @@ export function WorkflowEditor({
     publishedWorkflowId,
     handleShowSelector,
     handleCloneVersion,
+    useRag,
 }: {
     dataSources: WithStringId<z.infer<typeof DataSource>>[];
     workflow: WithStringId<z.infer<typeof Workflow>>;
     publishedWorkflowId: string | null;
     handleShowSelector: () => void;
     handleCloneVersion: (workflowId: string) => void;
+    useRag: boolean;
 }) {
     const [state, dispatch] = useReducer<Reducer<State, Action>>(reducer, {
         patches: [],
@@ -873,6 +875,7 @@ export function WorkflowEditor({
                     dataSources={dataSources}
                     handleUpdate={handleUpdateAgent.bind(null, state.present.selection.name)}
                     handleClose={handleUnselectAgent}
+                    useRag={useRag}
                 />}
                 {state.present.selection?.type === "tool" && <ToolConfig
                     key={state.present.selection.name}
diff --git a/apps/rowboat/app/scripts/rag_files_worker.ts b/apps/rowboat/app/scripts/rag_files_worker.ts
index 9fd962a9..509e5720 100644
--- a/apps/rowboat/app/scripts/rag_files_worker.ts
+++ b/apps/rowboat/app/scripts/rag_files_worker.ts
@@ -1,11 +1,8 @@
 import '../lib/loadenv';
-import FirecrawlApp from '@mendable/firecrawl-js';
 import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
 import { z } from 'zod';
 import { dataSourceDocsCollection, dataSourcesCollection } from '../lib/mongodb';
-import { EmbeddingRecord } from "../lib/types/datasource_types";
-import { DataSourceDoc } from "../lib/types/datasource_types";
-import { DataSource } from "../lib/types/datasource_types";
+import { EmbeddingRecord, DataSourceDoc, DataSource } from "../lib/types/datasource_types";
 import { WithId } from 'mongodb';
 import { embedMany } from 'ai';
 import { embeddingModel } from '../lib/embedding';
@@ -14,6 +11,7 @@ import { PrefixLogger } from "../lib/utils";
 import { GoogleGenerativeAI } from "@google/generative-ai";
 import { GetObjectCommand } from "@aws-sdk/client-s3";
 import { uploadsS3Client } from '../lib/uploads_s3_client';
+import crypto from 'crypto';
 
 const splitter = new RecursiveCharacterTextSplitter({
     separators: ['\n\n', '\n', '. ', '.', ''],
@@ -31,11 +29,11 @@ const genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY || '');
 
 async function getFileContent(s3Key: string): Promise<Buffer> {
     const command = new GetObjectCommand({
-        Bucket: process.env.UPLOADS_S3_BUCKET,
+        Bucket: process.env.RAG_UPLOADS_S3_BUCKET,
         Key: s3Key,
     });
     const response = await uploadsS3Client.send(command);
-    const chunks: Buffer[] = [];
+    const chunks: Uint8Array[] = [];
     for await (const chunk of response.Body as any) {
         chunks.push(chunk);
     }
diff --git a/apps/rowboat/app/scripts/rag_urls_worker.ts b/apps/rowboat/app/scripts/rag_urls_worker.ts
index cf533cab..c3597dc4 100644
--- a/apps/rowboat/app/scripts/rag_urls_worker.ts
+++ b/apps/rowboat/app/scripts/rag_urls_worker.ts
@@ -3,28 +3,16 @@ import FirecrawlApp from '@mendable/firecrawl-js';
 import { RecursiveCharacterTextSplitter } from "@langchain/textsplitters";
 import { z } from 'zod';
 import { dataSourceDocsCollection, dataSourcesCollection } from '../lib/mongodb';
-import { EmbeddingRecord } from "../lib/types/datasource_types";
-import { DataSourceDoc } from "../lib/types/datasource_types";
-import { DataSource } from "../lib/types/datasource_types";
+import { EmbeddingRecord, DataSourceDoc, DataSource } from "../lib/types/datasource_types";
 import { WithId } from 'mongodb';
 import { embedMany } from 'ai';
 import { embeddingModel } from '../lib/embedding';
 import { qdrantClient } from '../lib/qdrant';
 import { PrefixLogger } from "../lib/utils";
+import crypto from 'crypto';
 
 const firecrawl = new FirecrawlApp({ apiKey: process.env.FIRECRAWL_API_KEY });
 
-const firecrawlHttpAuth = {
-    'Authorization': `Bearer ${process.env.FIRECRAWL_API_KEY}`,
-}
-
-type Webpage = {
-    title: string,
-    url: string,
-    markdown: string,
-    html: string,
-}
-
 const splitter = new RecursiveCharacterTextSplitter({
     separators: ['\n\n', '\n', '. ', '.', ''],
     chunkSize: 1024,
@@ -36,18 +24,6 @@ const minute = 60 * second;
 const hour = 60 * minute;
 const day = 24 * hour;
 
-const firecrawlStatusPollInterval = 60 * second;
-
-/*
-const source: z.infer<typeof SourceSchema> = {
-  _id: new ObjectId(),
-  url: "https://www.example.com",
-  type: "web",
-  status: 'processing',
-  createdAt: new Date().toISOString(),
-};
-*/
-
 async function retryable<T>(fn: () => Promise<T>, maxAttempts: number = 3): Promise<T> {
     let attempts = 0;
     while (true) {
diff --git a/apps/rowboat/scripts.Dockerfile b/apps/rowboat/scripts.Dockerfile
new file mode 100644
index 00000000..01e1f338
--- /dev/null
+++ b/apps/rowboat/scripts.Dockerfile
@@ -0,0 +1,46 @@
+# syntax=docker.io/docker/dockerfile:1
+
+FROM node:18-alpine AS base
+
+# Install dependencies only when needed
+FROM base AS deps
+# Check https://github.com/nodejs/docker-node/tree/b4117f9333da4138b03a546ec926ef50a31506c3#nodealpine to understand why libc6-compat might be needed.
+RUN apk add --no-cache libc6-compat
+WORKDIR /app
+
+# Install dependencies based on the preferred package manager
+COPY package.json yarn.lock* package-lock.json* pnpm-lock.yaml* .npmrc* ./
+RUN \
+  if [ -f yarn.lock ]; then yarn --frozen-lockfile; \
+  elif [ -f package-lock.json ]; then npm ci; \
+  elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm i --frozen-lockfile; \
+  else echo "Lockfile not found." && exit 1; \
+  fi
+
+
+# Rebuild the source code only when needed
+FROM base AS builder
+WORKDIR /app
+COPY --from=deps /app/node_modules ./node_modules
+COPY . .
+
+# Next.js collects completely anonymous telemetry data about general usage.
+# Learn more here: https://nextjs.org/telemetry
+# Uncomment the following line in case you want to disable telemetry during the build.
+# ENV NEXT_TELEMETRY_DISABLED=1
+
+RUN \
+  if [ -f yarn.lock ]; then yarn run build; \
+  elif [ -f package-lock.json ]; then npm run build; \
+  elif [ -f pnpm-lock.yaml ]; then corepack enable pnpm && pnpm run build; \
+  else echo "Lockfile not found." && exit 1; \
+  fi
+
+ENV NODE_ENV=production
+# Uncomment the following line in case you want to disable telemetry during runtime.
+# ENV NEXT_TELEMETRY_DISABLED=1
+
+RUN addgroup --system --gid 1001 nodejs
+RUN adduser --system --uid 1001 nextjs
+
+USER nextjs
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index 311a93dc..cb026d57 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,23 +10,31 @@ services:
     environment:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
       - MONGODB_CONNECTION_STRING=${MONGODB_CONNECTION_STRING}
-      - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY}
-      - OXYLABS_USERNAME=${OXYLABS_USERNAME}
-      - OXYLABS_PASSWORD=${OXYLABS_PASSWORD}
-      - CHAT_WIDGET_SESSION_JWT_SECRET=${CHAT_WIDGET_SESSION_JWT_SECRET}
-      - AGENTS_API_URL=http://agents:3001
-      - AGENTS_API_KEY=${AGENTS_API_KEY}
-      - COPILOT_API_URL=http://copilot:3002
-      - COPILOT_API_KEY=${COPILOT_API_KEY}
       - AUTH0_SECRET=${AUTH0_SECRET}
       - AUTH0_BASE_URL=${AUTH0_BASE_URL}
       - AUTH0_ISSUER_BASE_URL=${AUTH0_ISSUER_BASE_URL}
       - AUTH0_CLIENT_ID=${AUTH0_CLIENT_ID}
       - AUTH0_CLIENT_SECRET=${AUTH0_CLIENT_SECRET}
+      - AGENTS_API_URL=http://agents:3001
+      - AGENTS_API_KEY=${AGENTS_API_KEY}
+      - COPILOT_API_URL=http://copilot:3002
+      - COPILOT_API_KEY=${COPILOT_API_KEY}
       - REDIS_URL=redis://redis:6379
+      - USE_RAG=${USE_RAG}
+      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_API_KEY=${QDRANT_API_KEY}
+      - USE_RAG_UPLOADS=${USE_RAG_UPLOADS}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - RAG_UPLOADS_S3_BUCKET=${RAG_UPLOADS_S3_BUCKET}
+      - RAG_UPLOADS_S3_REGION=${RAG_UPLOADS_S3_REGION}
+      - USE_RAG_SCRAPING=${USE_RAG_SCRAPING}
+      - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY}
+      - USE_CHAT_WIDGET=${USE_CHAT_WIDGET}
+      - CHAT_WIDGET_HOST=http://localhost:3006
+      - CHAT_WIDGET_SESSION_JWT_SECRET=${CHAT_WIDGET_SESSION_JWT_SECRET}
       - MAX_QUERIES_PER_MINUTE=${MAX_QUERIES_PER_MINUTE}
       - MAX_PROJECTS_PER_USER=${MAX_PROJECTS_PER_USER}
-      - CHAT_WIDGET_HOST=${CHAT_WIDGET_HOST}
     restart: unless-stopped
 
   agents:
@@ -51,16 +59,6 @@ services:
       - API_KEY=${COPILOT_API_KEY}
     restart: unless-stopped
 
-  tools_webhook:
-    build:
-      context: ./apps/tools_webhook
-      dockerfile: Dockerfile
-    ports:
-      - "3005:3005"
-    environment:
-      - SIGNING_SECRET=${SIGNING_SECRET}
-    restart: unless-stopped 
-
   simulation_runner:
     build:
       context: ./apps/simulation_runner
@@ -71,24 +69,60 @@ services:
       - OPENAI_API_KEY=${OPENAI_API_KEY}
     restart: unless-stopped
 
-  docs:
+  rag_files_worker:
     build:
-      context: ./apps/docs
-      dockerfile: Dockerfile
-    ports:
-      - "8000:8000"
+      context: ./apps/rowboat
+      dockerfile: scripts.Dockerfile
+    command: ["npm", "run", "ragFilesWorker"]
+    profiles: [ "rag_files_worker" ]
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - MONGODB_CONNECTION_STRING=${MONGODB_CONNECTION_STRING}
+      - GOOGLE_API_KEY=${GOOGLE_API_KEY}
+      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
+      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
+      - RAG_UPLOADS_S3_BUCKET=${RAG_UPLOADS_S3_BUCKET}
+      - RAG_UPLOADS_S3_REGION=${RAG_UPLOADS_S3_REGION}
+      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_API_KEY=${QDRANT_API_KEY}
     restart: unless-stopped
-  
+
+  rag_urls_worker:
+    build:
+      context: ./apps/rowboat
+      dockerfile: scripts.Dockerfile
+    command: ["npm", "run", "ragUrlsWorker"]
+    profiles: [ "rag_urls_worker" ]
+    environment:
+      - OPENAI_API_KEY=${OPENAI_API_KEY}
+      - MONGODB_CONNECTION_STRING=${MONGODB_CONNECTION_STRING}
+      - FIRECRAWL_API_KEY=${FIRECRAWL_API_KEY}
+      - QDRANT_URL=${QDRANT_URL}
+      - QDRANT_API_KEY=${QDRANT_API_KEY}
+    restart: unless-stopped
+
+  tools_webhook:
+    build:
+      context: ./apps/tools_webhook
+      dockerfile: Dockerfile
+    profiles: [ "tools_webhook" ]
+    ports:
+      - "3005:3005"
+    environment:
+      - SIGNING_SECRET=${SIGNING_SECRET}
+    restart: unless-stopped
+
   chat_widget:
     build:
       context: ./apps/chat_widget
       dockerfile: Dockerfile
+    profiles: [ "chat_widget" ]
     ports:
       - "3006:3006"
     environment:
       - PORT=3006
-      - CHAT_WIDGET_HOST=${CHAT_WIDGET_HOST}
-      - ROWBOAT_HOST=${ROWBOAT_HOST}
+      - CHAT_WIDGET_HOST=http://localhost:3006
+      - ROWBOAT_HOST=http://localhost:3000
     restart: unless-stopped
 
   redis:
@@ -96,3 +130,12 @@ services:
     ports:
       - "6379:6379"
     restart: unless-stopped
+
+  docs:
+    build:
+      context: ./apps/docs
+      dockerfile: Dockerfile
+    profiles: [ "docs" ]
+    ports:
+      - "8000:8000"
+    restart: unless-stopped