From 695ffca5fa8bd01ad32a81a98d3b01a68566290d Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 14:42:44 +0800 Subject: [PATCH 1/5] :sparkles: Add bing search engine --- metagpt/tools/__init__.py | 1 + metagpt/tools/search_engine.py | 3 + metagpt/tools/search_engine_bing.py | 103 ++++++++++++++++++++++ tests/metagpt/tools/test_search_engine.py | 1 + 4 files changed, 108 insertions(+) create mode 100644 metagpt/tools/search_engine_bing.py diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index 8d265e9f3..4b27be287 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -19,6 +19,7 @@ class SearchEngineType(Enum): DIRECT_GOOGLE = "google" DUCK_DUCK_GO = "ddg" CUSTOM_ENGINE = "custom" + Bing = "bing" class WebBrowserEngineType(Enum): diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 1e540bd0e..7c463605c 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -88,6 +88,9 @@ class SearchEngine(BaseModel): run_func = importlib.import_module(module).DDGAPIWrapper(**kwargs).run elif self.engine == SearchEngineType.CUSTOM_ENGINE: run_func = self.run_func + elif self.engine == SearchEngineType.Bing: + module = "metagpt.tools.search_engine_bing" + run_func = importlib.import_module(module).BingAPIWrapper(**kwargs).run else: raise NotImplementedError self.run_func = run_func diff --git a/metagpt/tools/search_engine_bing.py b/metagpt/tools/search_engine_bing.py new file mode 100644 index 000000000..831cf1648 --- /dev/null +++ b/metagpt/tools/search_engine_bing.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from __future__ import annotations + +import json +import warnings +from typing import Optional + +import aiohttp + +from pydantic import BaseModel, ConfigDict, model_validator + + +class BingAPIWrapper(BaseModel): + model_config = ConfigDict(arbitrary_types_allowed=True) + + api_key: str + bing_url: str = "https://api.bing.microsoft.com/v7.0/search" + aiosession: Optional[aiohttp.ClientSession] = None + proxy: Optional[str] = None + + @model_validator(mode="before") + @classmethod + def validate_api_key(cls, values: dict) -> dict: + if "api_key" in values: + values.setdefault("api_key", values["api_key"]) + warnings.warn("`api_key` is deprecated, use `api_key` instead", DeprecationWarning, stacklevel=2) + return values + + @property + def header(self): + return {"Ocp-Apim-Subscription-Key": self.api_key} + + async def run( + self, + query: str, + max_results: int = 8, + as_string: bool = True, + focus: list[str] | None = None, + ) -> str | list[dict]: + """Return the results of a Google search using the official Bing API. + + Args: + query: The search query. + max_results: The number of results to return. + as_string: A boolean flag to determine the return type of the results. If True, the function will + return a formatted string with the search results. If False, it will return a list of dictionaries + containing detailed information about each search result. + focus: Specific information to be focused on from each search result. + + Returns: + The results of the search. + """ + params = { + "q": query, + "count": max_results, + "textFormat": "HTML", + } + result = await self.results(params) + search_results = result["webPages"]["value"] + focus = focus or ["snippet", "url", "name"] + details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results] + if as_string: + return safe_results(details) + return details + + async def results(self, params: dict) -> dict: + """Use aiohttp to run query and return the results async.""" + + if not self.aiosession: + async with aiohttp.ClientSession() as session: + async with session.get(self.bing_url, params=params, headers=self.header, proxy=self.proxy) as response: + response.raise_for_status() + res = await response.json() + else: + async with self.aiosession.get(self.bing_url, params=params, headers=self.header, + proxy=self.proxy) as response: + response.raise_for_status() + res = await response.json() + + return res + + +def safe_results(results: str | list) -> str: + """Return the results of a bing search in a safe format. + + Args: + results: The search results. + + Returns: + The results of the search. + """ + if isinstance(results, list): + safe_message = json.dumps([result for result in results]) + else: + safe_message = results.encode("utf-8", "ignore").decode("utf-8") + return safe_message + + +if __name__ == "__main__": + import fire + + fire.Fire(BingAPIWrapper().run) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index 964ead02f..4877e250b 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -37,6 +37,7 @@ class MockSearchEnine: (SearchEngineType.SERPER_GOOGLE, None, 6, False), (SearchEngineType.DUCK_DUCK_GO, None, 8, True), (SearchEngineType.DUCK_DUCK_GO, None, 6, False), + (SearchEngineType.Bing, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), ], From ba6fa497a9d2eea4ae207012d6bba900b9e7f2ca Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 15:38:16 +0800 Subject: [PATCH 2/5] :sparkles: Add bing search engine --- metagpt/tools/search_engine_bing.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/tools/search_engine_bing.py b/metagpt/tools/search_engine_bing.py index 831cf1648..4ca9ce32d 100644 --- a/metagpt/tools/search_engine_bing.py +++ b/metagpt/tools/search_engine_bing.py @@ -58,7 +58,10 @@ class BingAPIWrapper(BaseModel): } result = await self.results(params) search_results = result["webPages"]["value"] - focus = focus or ["snippet", "url", "name"] + focus = focus or ["snippet", "link", "title"] + for item_dict in search_results: + item_dict["link"] = item_dict["url"] + item_dict["title"] = item_dict["name"] details = [{i: j for i, j in item_dict.items() if i in focus} for item_dict in search_results] if as_string: return safe_results(details) From b828f9d8ed911937d528462742ee2b2d2f40912b Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 21:53:51 +0800 Subject: [PATCH 3/5] Update __init__.py. BING="bing" --- metagpt/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index 4b27be287..35fa04658 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -19,7 +19,7 @@ class SearchEngineType(Enum): DIRECT_GOOGLE = "google" DUCK_DUCK_GO = "ddg" CUSTOM_ENGINE = "custom" - Bing = "bing" + BING = "bing" class WebBrowserEngineType(Enum): From 3d555cabc1b87ba13ffe34f1cb7f293dad8fea45 Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 21:54:34 +0800 Subject: [PATCH 4/5] Update test_search_engine.py BING="bing" --- tests/metagpt/tools/test_search_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index 4877e250b..498d3974d 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -37,7 +37,7 @@ class MockSearchEnine: (SearchEngineType.SERPER_GOOGLE, None, 6, False), (SearchEngineType.DUCK_DUCK_GO, None, 8, True), (SearchEngineType.DUCK_DUCK_GO, None, 6, False), - (SearchEngineType.Bing, None, 6, False), + (SearchEngineType.BING, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), ], From 9793f08d071ab38a5d7dc608400a895b81d69d06 Mon Sep 17 00:00:00 2001 From: wiley Date: Wed, 27 Mar 2024 21:57:24 +0800 Subject: [PATCH 5/5] Update search_engine.py BING="bing" --- metagpt/tools/search_engine.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 7c463605c..767f4aaba 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -88,7 +88,7 @@ class SearchEngine(BaseModel): run_func = importlib.import_module(module).DDGAPIWrapper(**kwargs).run elif self.engine == SearchEngineType.CUSTOM_ENGINE: run_func = self.run_func - elif self.engine == SearchEngineType.Bing: + elif self.engine == SearchEngineType.BING: module = "metagpt.tools.search_engine_bing" run_func = importlib.import_module(module).BingAPIWrapper(**kwargs).run else: