diff --git a/apps/rowboat/app/actions.ts b/apps/rowboat/app/actions.ts index fef6f4ab..af06afc0 100644 --- a/apps/rowboat/app/actions.ts +++ b/apps/rowboat/app/actions.ts @@ -435,8 +435,18 @@ export async function createCrawlDataSource(projectId: string, formData: FormDat export async function createUrlsDataSource(projectId: string, formData: FormData) { await projectAuthCheck(projectId); const urls = formData.get('urls') as string; - // take first 100 urls - const limitedUrls = urls.split('\n').slice(0, 100).map((url) => url.trim()); + // take first 100 valid urls (as in parse them) + const limitedUrls = urls.split('\n') + .map((url) => url.trim()) + .filter((url) => { + try { + new URL(url); + return true; + } catch (e) { + return false; + } + }) + .slice(0, 100); const name = formData.get('name') as string; const result = await dataSourcesCollection.insertOne({ diff --git a/apps/rowboat/app/projects/[projectId]/layout.tsx b/apps/rowboat/app/projects/[projectId]/layout.tsx index 6b0bb0a9..108d21bd 100644 --- a/apps/rowboat/app/projects/[projectId]/layout.tsx +++ b/apps/rowboat/app/projects/[projectId]/layout.tsx @@ -7,8 +7,10 @@ export default async function Layout({ params: { projectId: string } children: React.ReactNode }) { + const useDataSources = process.env.USE_DATA_SOURCES === 'true'; + return
-
; } \ No newline at end of file diff --git a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx index e9c0ba5f..d47cb9a3 100644 --- a/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx +++ b/apps/rowboat/app/projects/[projectId]/sources/new/form.tsx @@ -10,9 +10,9 @@ export function Form({ }: { projectId: string; }) { - const [sourceType, setSourceType] = useState("crawl"); + const [sourceType, setSourceType] = useState(""); - const createCrawlDataSourceWithProjectId = createCrawlDataSource.bind(null, projectId); + // const createCrawlDataSourceWithProjectId = createCrawlDataSource.bind(null, projectId); const createUrlsDataSourceWithProjectId = createUrlsDataSource.bind(null, projectId); function handleSourceTypeChange(event: React.ChangeEvent) { @@ -26,22 +26,23 @@ export function Form({ selectedKeys={[sourceType]} onChange={handleSourceTypeChange} > - } > Crawl URLs - + */} } > - Specify URLs + Scrape URLs - {sourceType === "crawl" &&
@@ -95,7 +96,7 @@ export function Form({ , }} /> -
} + } */} {sourceType === "urls" &&
Note: