From a12569234597b8ffec9b5a0c275af57b24c4f52d Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 28 Dec 2023 18:45:46 +0800 Subject: [PATCH 1/3] add test extras_require --- setup.py | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/setup.py b/setup.py index 2163b4233..b69f05b45 100644 --- a/setup.py +++ b/setup.py @@ -22,6 +22,29 @@ here = Path(__file__).resolve().parent long_description = (here / "README.md").read_text(encoding="utf-8") requirements = (here / "requirements.txt").read_text(encoding="utf-8").splitlines() + +extras_require = { + "playwright": ["playwright>=1.26", "beautifulsoup4"], + "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], + "search-google": ["google-api-python-client==2.94.0"], + "search-ddg": ["duckduckgo-search==3.8.5"], + "pyppeteer": ["pyppeteer>=1.0.2"], + "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], + "test": ["pytest", "pytest-cov", "pytest-asyncio", "pytest-mock"], +} + +extras_require["test"] = [ + *set(i for j in extras_require.values() for i in j), + "pytest", + "pytest-asyncio", + "pytest-cov", + "pytest-mock", + "pytest-html", +] + +extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pre-commit~=3.6.0"],) + + setup( name="metagpt", version="0.5.2", @@ -36,16 +59,7 @@ setup( packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, - extras_require={ - "playwright": ["playwright>=1.26", "beautifulsoup4"], - "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], - "search-google": ["google-api-python-client==2.94.0"], - "search-ddg": ["duckduckgo-search==3.8.5"], - "pyppeteer": ["pyppeteer>=1.0.2"], - "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], - "dev": ["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pre-commit~=3.6.0"], - "test": ["pytest", "pytest-cov", "pytest-asyncio", "pytest-mock"], - }, + extras_require=extras_require, cmdclass={ "install_mermaid": InstallMermaidCLI, }, From 8cfb031a7294b47afab3faab876cb6664c194af1 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 28 Dec 2023 22:34:28 +0800 Subject: [PATCH 2/3] add proxy for webdriver downloader --- metagpt/tools/web_browser_engine_selenium.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index 8bc81f956..70b651935 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -14,6 +14,8 @@ from typing import Literal from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait +from webdriver_manager.core.download_manager import WDMDownloadManager +from webdriver_manager.core.http import WDMHttpClient from metagpt.config import CONFIG from metagpt.utils.parse_html import WebPage @@ -93,6 +95,13 @@ _webdriver_manager_types = { } +class WDMHttpProxyClient(WDMHttpClient): + def get(self, url, **kwargs): + if "proxies" not in kwargs and CONFIG.global_proxy: + kwargs["proxies"] = {"all_proxy": CONFIG.global_proxy} + return super().get(url, **kwargs) + + def _gen_get_driver_func(browser_type, *args, executable_path=None): WebDriver = getattr(importlib.import_module(f"selenium.webdriver.{browser_type}.webdriver"), "WebDriver") Service = getattr(importlib.import_module(f"selenium.webdriver.{browser_type}.service"), "Service") @@ -101,7 +110,7 @@ def _gen_get_driver_func(browser_type, *args, executable_path=None): if not executable_path: module_name, type_name = _webdriver_manager_types[browser_type] DriverManager = getattr(importlib.import_module(module_name), type_name) - driver_manager = DriverManager() + driver_manager = DriverManager(download_manager=WDMDownloadManager(http_client=WDMHttpProxyClient())) # driver_manager.driver_cache.find_driver(driver_manager.driver)) executable_path = driver_manager.install() From ca7d54696d1f57e0902bbde196ac427c674ea641 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Thu, 28 Dec 2023 22:47:03 +0800 Subject: [PATCH 3/3] update the pyppeteer extras require --- setup.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b69f05b45..4c2941a18 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,6 @@ extras_require = { "selenium": ["selenium>4", "webdriver_manager", "beautifulsoup4"], "search-google": ["google-api-python-client==2.94.0"], "search-ddg": ["duckduckgo-search==3.8.5"], - "pyppeteer": ["pyppeteer>=1.0.2"], "ocr": ["paddlepaddle==2.4.2", "paddleocr>=2.0.1", "tabulate==0.9.0"], "test": ["pytest", "pytest-cov", "pytest-asyncio", "pytest-mock"], } @@ -42,6 +41,9 @@ extras_require["test"] = [ "pytest-html", ] +extras_require["pyppeteer"] = [ + "pyppeteer>=1.0.2" +] # pyppeteer is unmaintained and there are conflicts with dependencies extras_require["dev"] = (["pylint~=3.0.3", "black~=23.3.0", "isort~=5.12.0", "pre-commit~=3.6.0"],)