From a1adbd722c6b1bba70c79f96b66d4f0cf6bf8bc6 Mon Sep 17 00:00:00 2001
From: ramnique <30795890+ramnique@users.noreply.github.com>
Date: Wed, 5 Feb 2025 18:12:34 +0530
Subject: [PATCH] use only firecrawl for scraping, dont rewrite

---
 apps/rowboat/app/scripts/crawlUrls.ts | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/apps/rowboat/app/scripts/crawlUrls.ts b/apps/rowboat/app/scripts/crawlUrls.ts
index 62536ef0..02474d77 100644
--- a/apps/rowboat/app/scripts/crawlUrls.ts
+++ b/apps/rowboat/app/scripts/crawlUrls.ts
@@ -685,6 +685,18 @@ async function saveWebpagesToMongodb(logger: PrefixLogger, job: WithId<z.infer<t
     });
 }
 
+async function dummyRewrite(_logger: PrefixLogger) {
+    const logger = _logger.child('dummyRewrite');
+    await batchMode({
+        inputFilePath: 'crawled.jsonl',
+        outputFilePath: 'rewritten.jsonl',
+        batchSize: 10,
+        processBatch: async (batch: Webpage[]): Promise<Webpage[]> => {
+            return batch;
+        }
+    });
+}
+
 async function rewrite(_logger: PrefixLogger) {
     const logger = _logger.child('rewrite');
 
@@ -950,7 +962,8 @@ async function mongodb(logger: PrefixLogger, job: WithId<z.infer<typeof DataSour
                 await dataSourcesCollection.updateOne({ _id: job._id }, { $set: { 'data.scrapedUrls': Array.from(crawledUrls).join('\n') } });
             }
             // rewrite the merged results as simplified html and markdown
-            await rewrite(logger);
+            // await rewrite(logger);
+            await dummyRewrite(logger);
             await saveWebpagesToMongodb(logger, job);
             await chunk(logger, job);
             await embeddings(logger);