update timeout

metagpt/tools/web_browser_engine.py

@@ -92,14 +92,14 @@ class WebBrowserEngine(BaseModel):
         return cls(**data, **kwargs)
 
     @overload
-    async def run(self, url: str) -> WebPage:
+    async def run(self, url: str, per_page_timeout: float = None) -> WebPage:
         ...
 
     @overload
-    async def run(self, url: str, *urls: str) -> list[WebPage]:
+    async def run(self, url: str, *urls: str, per_page_timeout: float = None) -> list[WebPage]:
         ...
 
-    async def run(self, url: str, *urls: str) -> WebPage | list[WebPage]:
+    async def run(self, url: str, *urls: str, per_page_timeout: float = None) -> WebPage | list[WebPage]:
         """Runs the browser engine to load one or more web pages.
 
         This method is the implementation of the overloaded run signatures. It delegates the task
@@ -108,8 +108,9 @@ class WebBrowserEngine(BaseModel):
         Args:
             url: The URL of the first web page to load.
             *urls: Additional URLs of web pages to load, if any.
+            per_page_timeout: The maximum time for fetching a single page in seconds.
 
         Returns:
             A WebPage object if a single URL is provided, or a list of WebPage objects if multiple URLs are provided.
         """
-        return await self.run_func(url, *urls)
+        return await self.run_func(url, *urls, per_page_timeout=per_page_timeout)

metagpt/tools/web_browser_engine_playwright.py

@@ -42,7 +42,10 @@ class PlaywrightWrapper(BaseModel):
         if "ignore_https_errors" in kwargs:
             self.context_kwargs["ignore_https_errors"] = kwargs["ignore_https_errors"]
 
-    async def run(self, url: str, *urls: str) -> WebPage | list[WebPage]:
+        if "java_script_enabled" in kwargs:
+            self.context_kwargs["java_script_enabled"] = kwargs["java_script_enabled"]
+
+    async def run(self, url: str, *urls: str, per_page_timeout: float = None) -> WebPage | list[WebPage]:
         async with async_playwright() as ap:
             browser_type = getattr(ap, self.browser_type)
             await self._run_precheck(browser_type)
@@ -50,11 +53,17 @@ class PlaywrightWrapper(BaseModel):
             _scrape = self._scrape
 
             if urls:
-                return await asyncio.gather(_scrape(browser, url), *(_scrape(browser, i) for i in urls))
-            return await _scrape(browser, url)
+                return await asyncio.gather(
+                    _scrape(browser, url, per_page_timeout), *(_scrape(browser, i, per_page_timeout) for i in urls)
+                )
+            return await _scrape(browser, url, per_page_timeout)
 
-    async def _scrape(self, browser, url):
+    async def _scrape(self, browser, url, timeout: float = None):
         context = await browser.new_context(**self.context_kwargs)
+
+        if timeout is not None:
+            context.set_default_timeout(timeout * 1000)  # playwright uses milliseconds.
+
         page = await context.new_page()
         async with page:
             try:

metagpt/tools/web_browser_engine_selenium.py

@@ -54,14 +54,16 @@ class SeleniumWrapper(BaseModel):
     def executable_path(self):
         return self.launch_kwargs.get("executable_path")
 
-    async def run(self, url: str, *urls: str) -> WebPage | list[WebPage]:
+    async def run(self, url: str, *urls: str, per_page_timeout: float = None) -> WebPage | list[WebPage]:
         await self._run_precheck()
 
-        _scrape = lambda url: self.loop.run_in_executor(self.executor, self._scrape_website, url)
+        _scrape = lambda url, per_page_timeout: self.loop.run_in_executor(
+            self.executor, self._scrape_website, url, per_page_timeout
+        )
 
         if urls:
-            return await asyncio.gather(_scrape(url), *(_scrape(i) for i in urls))
-        return await _scrape(url)
+            return await asyncio.gather(_scrape(url, per_page_timeout), *(_scrape(i, per_page_timeout) for i in urls))
+        return await _scrape(url, per_page_timeout)
 
     async def _run_precheck(self):
         if self._has_run_precheck:
@@ -75,11 +77,11 @@ class SeleniumWrapper(BaseModel):
         )
         self._has_run_precheck = True
 
-    def _scrape_website(self, url):
+    def _scrape_website(self, url, timeout: float = None):
         with self._get_driver() as driver:
             try:
                 driver.get(url)
-                WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
+                WebDriverWait(driver, timeout or 30).until(EC.presence_of_element_located((By.TAG_NAME, "body")))
                 inner_text = driver.execute_script("return document.body.innerText;")
                 html = driver.page_source
             except Exception as e: