From b4536807d71349e08e50a9c8f51e72e5f7ff239c Mon Sep 17 00:00:00 2001 From: leonzh0u Date: Fri, 21 Jul 2023 09:42:01 -0400 Subject: [PATCH 1/2] fix serper integration bug to support batch queries --- examples/search_with_specific_engine.py | 2 +- metagpt/roles/role.py | 2 ++ metagpt/tools/search_engine.py | 7 ++----- metagpt/tools/search_engine_serper.py | 3 ++- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 81333bf83..b4da84f47 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -5,7 +5,7 @@ from metagpt.tools import SearchEngineType async def main(): # Serper API - await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run("What are some good sun protection products?") + await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) # Serper API #await Searcher(engine = SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") # Google API diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 36269aed2..e8870b4d1 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -221,6 +221,8 @@ class Role: message = Message(message) if isinstance(message, Message): self.recv(message) + if isinstance(message, list): + self.recv(Message("|".join(message))) elif not await self._observe(): # 如果没有任何新信息，挂起等待 logger.debug(f"{self._setting}: no news. waiting.") diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 5b9e1cd23..fb666f952 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -39,7 +39,7 @@ class SearchEngine: logger.info(results) return results - async def run(self, query, max_results=8): + async def run(self, query: str, max_results=8): if self.engine == SearchEngineType.SERPAPI_GOOGLE: api = SerpAPIWrapper() rsp = await api.run(query) @@ -47,10 +47,7 @@ class SearchEngine: rsp = SearchEngine.run_google(query, max_results) elif self.engine == SearchEngineType.SERPER_GOOGLE: api = SerperWrapper() - if isinstance(query, list): - rsp = await api.run(query) - elif isinstance(query, str): - rsp = await api.run([query]) + rsp = await api.run(query) elif self.engine == SearchEngineType.CUSTOM_ENGINE: rsp = self.run_func(query) else: diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 91a8afce9..2d105326d 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -38,7 +38,8 @@ class SerperWrapper(BaseModel): async def run(self, query: str, **kwargs: Any) -> str: """Run query through Serper and parse result async.""" - return ";".join([self._process_response(res) for res in await self.results(query)]) + queries = query.split("|") + return "|".join([self._process_response(res) for res in await self.results(queries)]) async def results(self, queries: list[str]) -> dict: """Use aiohttp to run query through Serper and return the results async.""" From 3e13a00003afebaf1fb6c1c38f4b3e490c873c97 Mon Sep 17 00:00:00 2001 From: leonzh0u Date: Sun, 23 Jul 2023 18:25:08 -0400 Subject: [PATCH 2/2] fix batch query: --- examples/search_with_specific_engine.py | 8 ++++---- metagpt/roles/role.py | 2 +- metagpt/tools/search_engine.py | 26 +++++++++++-------------- metagpt/tools/search_engine_serper.py | 4 ++-- 4 files changed, 18 insertions(+), 22 deletions(-) diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index c91769859..7cc431cd4 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -6,11 +6,11 @@ from metagpt.tools import SearchEngineType async def main(): # Serper API - await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) - # Serper API - # await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") + #await Searcher(engine = SearchEngineType.SERPER_GOOGLE).run(["What are some good sun protection products?","What are some of the best beaches?"]) + # SerpAPI + #await Searcher(engine=SearchEngineType.SERPAPI_GOOGLE).run("What are the best ski brands for skiers?") # Google API - # await Searcher(engine=SearchEngineType.DIRECT_GOOGLE).run("What are the most interesting human facts?") + await Searcher(engine=SearchEngineType.DIRECT_GOOGLE).run("What are the most interesting human facts?") if __name__ == '__main__': asyncio.run(main()) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1ce85491d..e99b0d199 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -226,7 +226,7 @@ class Role: if isinstance(message, Message): self.recv(message) if isinstance(message, list): - self.recv(Message("|".join(message))) + self.recv(Message("\n".join(message))) elif not await self._observe(): # 如果没有任何新信息，挂起等待 logger.debug(f"{self._setting}: no news. waiting.") diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 1f229a72a..cfd4e8789 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -53,11 +53,11 @@ class SearchEngine: return rsp -def google_official_search(queries: list[str], num_results: int = 8, focus=['snippet', 'link', 'title']) -> dict | list[dict]: - """Return the results of a batch of Google search using the official Google API +def google_official_search(query: str, num_results: int = 8, focus=['snippet', 'link', 'title']) -> dict | list[dict]: + """Return the results of a Google search using the official Google API Args: - queries (list[str]): A batch of search queries. + query (str): The search query. num_results (int): The number of results to return. Returns: @@ -71,19 +71,15 @@ def google_official_search(queries: list[str], num_results: int = 8, focus=['sni api_key = config.google_api_key custom_search_engine_id = config.google_cse_id - service = build("customsearch", "v2", developerKey=api_key) - batch = service.new_batch_http_request() - for query in queries: - batch.add(service.cse() - .list(q=query, cx=custom_search_engine_id, num=num_results)) - batch.execute() - result = ( - service.cse() - .list(q=query, cx=custom_search_engine_id, num=num_results) - .execute() - ) + with build("customsearch", "v1", developerKey=api_key) as service: - # Extract the search result items from the response + result = ( + service.cse() + .list(q=query, cx=custom_search_engine_id, num=num_results) + .execute() + ) + logger.info(result) + # Extract the search result items from the response search_results = result.get("items", []) # Create a list of only the URLs from the search results diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index fefaa1eb5..80c2f8001 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -38,8 +38,8 @@ class SerperWrapper(BaseModel): async def run(self, query: str, **kwargs: Any) -> str: """Run query through Serper and parse result async.""" - queries = query.split("|") - return "|".join([self._process_response(res) for res in await self.results(queries)]) + queries = query.split("\n") + return "\n".join([self._process_response(res) for res in await self.results(queries)]) async def results(self, queries: list[str]) -> dict: """Use aiohttp to run query through Serper and return the results async."""