diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index 8bc81f956..70b651935 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -14,6 +14,8 @@ from typing import Literal from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait +from webdriver_manager.core.download_manager import WDMDownloadManager +from webdriver_manager.core.http import WDMHttpClient from metagpt.config import CONFIG from metagpt.utils.parse_html import WebPage @@ -93,6 +95,13 @@ _webdriver_manager_types = { } +class WDMHttpProxyClient(WDMHttpClient): + def get(self, url, **kwargs): + if "proxies" not in kwargs and CONFIG.global_proxy: + kwargs["proxies"] = {"all_proxy": CONFIG.global_proxy} + return super().get(url, **kwargs) + + def _gen_get_driver_func(browser_type, *args, executable_path=None): WebDriver = getattr(importlib.import_module(f"selenium.webdriver.{browser_type}.webdriver"), "WebDriver") Service = getattr(importlib.import_module(f"selenium.webdriver.{browser_type}.service"), "Service") @@ -101,7 +110,7 @@ def _gen_get_driver_func(browser_type, *args, executable_path=None): if not executable_path: module_name, type_name = _webdriver_manager_types[browser_type] DriverManager = getattr(importlib.import_module(module_name), type_name) - driver_manager = DriverManager() + driver_manager = DriverManager(download_manager=WDMDownloadManager(http_client=WDMHttpProxyClient())) # driver_manager.driver_cache.find_driver(driver_manager.driver)) executable_path = driver_manager.install() diff --git a/setup.py b/setup.py index 2163b4233..4c2941a18 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,31 @@ here = Path(__file__).resolve().parent long_description = (here / "README.md").read_text(encoding="utf-8") requirements = (here / "requirements.txt").read_text(encoding="utf-8").splitlines() + +extras_require = { + "playwright": ["playwright>=1.26", "beautifulsoup4"], + "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], + "search-google": ["google-api-python-client==2.94.0"], + "search-ddg": ["duckduckgo-search==3.8.5"], + "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], + "test": ["pytest", "pytest-cov", "pytest-asyncio", "pytest-mock"], +} + +extras_require["test"] = [ + *set(i for j in extras_require.values() for i in j), + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "pytest-html", +] + +extras_require["pyppeteer"] = [ + "pyppeteer>=1.0.2" +] # pyppeteer is unmaintained and there are conflicts with dependencies +extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pre-commit~=3.6.0"],) + + setup( name="metagpt", version="0.5.2", @@ -36,16 +61,7 @@ setup( packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, - extras_require={ - "playwright": ["playwright>=1.26", "beautifulsoup4"], - "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], - "search-google": ["google-api-python-client==2.94.0"], - "search-ddg": ["duckduckgo-search==3.8.5"], - "pyppeteer": ["pyppeteer>=1.0.2"], - "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], - "dev": ["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pre-commit~=3.6.0"], - "test": ["pytest", "pytest-cov", "pytest-asyncio", "pytest-mock"], - }, + extras_require=extras_require, cmdclass={ "install_mermaid": InstallMermaidCLI, },