From 4b120a932f7821088021f71c21c4e32e6b3fca08 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 23 Dec 2023 21:56:19 +0800 Subject: [PATCH 1/9] add options to disable llm provider check --- config/config.yaml | 4 +++- metagpt/config.py | 4 +++- metagpt/llm.py | 7 ++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index a9c764c56..6a1fd597f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -117,4 +117,6 @@ RPM: 10 ### repair operation on the content extracted from LLM's raw output. Warning, it improves the result but not fix all cases. # REPAIR_LLM_OUTPUT: false -# PROMPT_FORMAT: json #json or markdown \ No newline at end of file +# PROMPT_FORMAT: json #json or markdown + +# DISABLE_LLM_PROVIDER_CHECK: false diff --git a/metagpt/config.py b/metagpt/config.py index 208b4fd7b..16df19a4c 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -107,7 +107,9 @@ class Config(metaclass=Singleton): self.gemini_api_key = self._get("GEMINI_API_KEY") self.ollama_api_base = self._get("OLLAMA_API_BASE") self.ollama_api_model = self._get("OLLAMA_API_MODEL") - _ = self.get_default_llm_provider_enum() + + if not self._get("DISABLE_LLM_PROVIDER_CHECK"): + _ = self.get_default_llm_provider_enum() self.openai_base_url = self._get("OPENAI_BASE_URL") self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy diff --git a/metagpt/llm.py b/metagpt/llm.py index 8763642f0..f1cb98dae 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -6,6 +6,8 @@ @File : llm.py """ +from typing import Optional + from metagpt.config import CONFIG, LLMProviderEnum from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.human_provider import HumanProvider @@ -14,6 +16,9 @@ from metagpt.provider.llm_provider_registry import LLM_REGISTRY _ = HumanProvider() # Avoid pre-commit error -def LLM(provider: LLMProviderEnum = CONFIG.get_default_llm_provider_enum()) -> BaseGPTAPI: +def LLM(provider: Optional[LLMProviderEnum] = None) -> BaseGPTAPI: """get the default llm provider""" + if provider is None: + provider = CONFIG.get_default_llm_provider_enum() + return LLM_REGISTRY.get_provider(provider) From 011ae46c09dd81f10e49060530f73985ec13c488 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 23 Dec 2023 21:58:54 +0800 Subject: [PATCH 2/9] Lazy Loading WEB_BROWSER_ENGINE --- metagpt/actions/research.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 074cdee0a..c47a77bdd 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -180,9 +180,11 @@ class WebBrowseAndSummarize(Action): llm: BaseGPTAPI = Field(default_factory=LLM) desc: str = "Explore the web and provide summaries of articles and webpages." browse_func: Union[Callable[[list[str]], None], None] = None - web_browser_engine: WebBrowserEngine = WebBrowserEngine( - engine=WebBrowserEngineType.CUSTOM if browse_func else None, - run_func=browse_func, + web_browser_engine: WebBrowserEngine = Field( + default_factory=lambda: WebBrowserEngine( + engine=WebBrowserEngineType.CUSTOM if WebBrowseAndSummarize.browse_func else None, + run_func=WebBrowseAndSummarize.browse_func, + ) ) def __init__(self, **kwargs): From b4552938e68de47fb9cc8af987b79403e5a146c6 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 23 Dec 2023 22:45:20 +0800 Subject: [PATCH 3/9] add llm stream log --- metagpt/logs.py | 13 +++++++++++++ metagpt/provider/google_gemini_api.py | 4 ++-- metagpt/provider/ollama_api.py | 4 ++-- metagpt/provider/openai_api.py | 4 ++-- metagpt/provider/zhipuai_api.py | 4 ++-- 5 files changed, 21 insertions(+), 8 deletions(-) diff --git a/metagpt/logs.py b/metagpt/logs.py index ab1bc4e94..fb0fdd553 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -8,6 +8,7 @@ import sys from datetime import datetime +from functools import partial from loguru import logger as _logger @@ -26,3 +27,15 @@ def define_log_level(print_level="INFO", logfile_level="DEBUG"): logger = define_log_level() + + +def log_llm_stream(msg): + _llm_stream_log(msg) + + +def set_llm_stream_logfunc(func): + global _llm_stream_log + _llm_stream_log = func + + +_llm_stream_log = partial(print, end="") diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 682f7b507..63a2ff687 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -20,7 +20,7 @@ from tenacity import ( ) from metagpt.config import CONFIG, LLMProviderEnum -from metagpt.logs import logger +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.llm_provider_registry import register_provider from metagpt.provider.openai_api import CostManager, log_and_reraise @@ -119,7 +119,7 @@ class GeminiGPTAPI(BaseGPTAPI): collected_content = [] async for chunk in resp: content = chunk.text - print(content, end="") + log_llm_stream(content, end="") collected_content.append(content) full_content = "".join(collected_content) diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.py index a15c46458..d668d3af1 100644 --- a/metagpt/provider/ollama_api.py +++ b/metagpt/provider/ollama_api.py @@ -15,7 +15,7 @@ from tenacity import ( from metagpt.config import CONFIG, LLMProviderEnum from metagpt.const import LLM_API_TIMEOUT -from metagpt.logs import logger +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.general_api_requestor import GeneralAPIRequestor from metagpt.provider.llm_provider_registry import register_provider @@ -127,7 +127,7 @@ class OllamaGPTAPI(BaseGPTAPI): if not chunk.get("done", False): content = self.get_choice_text(chunk) collected_content.append(content) - print(content, end="") + log_llm_stream(content, end="") else: # stream finished usage = self.get_usage(chunk) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 8fd8959b2..0b6fdd869 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -29,7 +29,7 @@ from tenacity import ( ) from metagpt.config import CONFIG, Config, LLMProviderEnum -from metagpt.logs import logger +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE from metagpt.provider.llm_provider_registry import register_provider @@ -222,7 +222,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): chunk_message = chunk.choices[0].delta # extract the message collected_messages.append(chunk_message) # save the message if chunk_message.content: - print(chunk_message.content, end="") + log_llm_stream(chunk_message.content) print() full_reply_content = "".join([m.content for m in collected_messages if m.content]) diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 60d9a0777..650720d6f 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -17,7 +17,7 @@ from tenacity import ( ) from metagpt.config import CONFIG, LLMProviderEnum -from metagpt.logs import logger +from metagpt.logs import log_llm_stream, logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.provider.llm_provider_registry import register_provider from metagpt.provider.openai_api import CostManager, log_and_reraise @@ -94,7 +94,7 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): if event.event == ZhiPuEvent.ADD.value: content = event.data collected_content.append(content) - print(content, end="") + log_llm_stream(content) elif event.event == ZhiPuEvent.ERROR.value or event.event == ZhiPuEvent.INTERRUPTED.value: content = event.data logger.error(f"event error: {content}", end="") From ddc0d3faa427ab94b1d10e94e609411731fbbbe3 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sun, 24 Dec 2023 03:52:29 +0800 Subject: [PATCH 4/9] not call LLM in global --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index c25cd947c..fe61b9878 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -152,7 +152,7 @@ class Role(BaseModel): builtin_class_name: str = "" _private_attributes = { - "_llm": LLM() if not is_human else HumanProvider(), + "_llm": None, "_role_id": _role_id, "_states": [], "_actions": [], From b113aa246f704326b87e1437dc8e2a41ef0d1ec7 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Mon, 25 Dec 2023 17:22:30 +0800 Subject: [PATCH 5/9] update log_llm_stream in log_llm_stream.py/ollama_api.py --- metagpt/provider/google_gemini_api.py | 2 +- metagpt/provider/ollama_api.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/google_gemini_api.py b/metagpt/provider/google_gemini_api.py index 63a2ff687..825b0bfe3 100644 --- a/metagpt/provider/google_gemini_api.py +++ b/metagpt/provider/google_gemini_api.py @@ -119,7 +119,7 @@ class GeminiGPTAPI(BaseGPTAPI): collected_content = [] async for chunk in resp: content = chunk.text - log_llm_stream(content, end="") + log_llm_stream(content) collected_content.append(content) full_content = "".join(collected_content) diff --git a/metagpt/provider/ollama_api.py b/metagpt/provider/ollama_api.py index d668d3af1..e913f3d0d 100644 --- a/metagpt/provider/ollama_api.py +++ b/metagpt/provider/ollama_api.py @@ -127,7 +127,7 @@ class OllamaGPTAPI(BaseGPTAPI): if not chunk.get("done", False): content = self.get_choice_text(chunk) collected_content.append(content) - log_llm_stream(content, end="") + log_llm_stream(content) else: # stream finished usage = self.get_usage(chunk) From 90bbf72ae8c8acb180b053c191fe7e531e448aff Mon Sep 17 00:00:00 2001 From: femto Date: Mon, 25 Dec 2023 17:35:13 +0800 Subject: [PATCH 6/9] fix sk agent --- metagpt/roles/sk_agent.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index 791dff5e2..6063205bd 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -7,13 +7,13 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.2.1 and 2.2.2 of RFC 116, utilize the new message distribution feature for message filtering. """ +from typing import Any, Type from pydantic import Field from semantic_kernel import Kernel -from semantic_kernel.orchestration.sk_function_base import SKFunctionBase from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from semantic_kernel.planning.basic_planner import BasicPlanner, Plan +from semantic_kernel.planning.basic_planner import BasicPlanner from metagpt.actions import UserRequirement from metagpt.actions.execute_task import ExecuteTask @@ -41,13 +41,13 @@ class SkAgent(Role): goal: str = "Execute task based on passed in task description" constraints: str = "" - plan: Plan = None - planner_cls: BasicPlanner = BasicPlanner - planner: BasicPlanner = Field(default_factory=BasicPlanner) + plan: Any = None + planner_cls: Any = None + planner: Any = None llm: BaseGPTAPI = Field(default_factory=LLM) kernel: Kernel = Field(default_factory=Kernel) - import_semantic_skill_from_directory: str = "" - import_skill: dict[str, SKFunctionBase] = dict() + import_semantic_skill_from_directory: Type[Kernel.import_semantic_skill_from_directory] = None + import_skill: Type[Kernel.import_skill] = None def __init__(self, **kwargs) -> None: """Initializes the Engineer role with given attributes.""" @@ -57,8 +57,8 @@ class SkAgent(Role): self.kernel = make_sk_kernel() # how funny the interface is inconsistent - if self.planner_cls == BasicPlanner: - self.planner = self.planner_cls() + if self.planner_cls == BasicPlanner or self.planner_cls is None: + self.planner = BasicPlanner() elif self.planner_cls in [SequentialPlanner, ActionPlanner]: self.planner = self.planner_cls(self.kernel) else: @@ -78,6 +78,7 @@ class SkAgent(Role): async def _act(self) -> Message: # how funny the interface is inconsistent + result = None if isinstance(self.planner, BasicPlanner): result = await self.planner.execute_plan_async(self.plan, self.kernel) elif any(isinstance(self.planner, cls) for cls in [SequentialPlanner, ActionPlanner]): From a87b5056d707c4efde11104768ff9c0702dbed9f Mon Sep 17 00:00:00 2001 From: xiaofenggang Date: Mon, 25 Dec 2023 16:04:58 +0000 Subject: [PATCH 7/9] [Bugfix] Set openai proxy for class ZhiPuAPTAPI When using ZHIPUAI as the large model provider, it is not possible to access ZHIPUAI in an HTTP proxy environment, and the following error will be reported: openai.error.APIConnectionError: Error communicating with OpenAI So we need set proxy for class ZhiPuAPTAPI. --- metagpt/provider/zhipuai_api.py | 2 ++ tests/metagpt/provider/test_zhipuai_api.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/metagpt/provider/zhipuai_api.py b/metagpt/provider/zhipuai_api.py index 650720d6f..b258d2883 100644 --- a/metagpt/provider/zhipuai_api.py +++ b/metagpt/provider/zhipuai_api.py @@ -50,6 +50,8 @@ class ZhiPuAIGPTAPI(BaseGPTAPI): assert config.zhipuai_api_key zhipuai.api_key = config.zhipuai_api_key openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used. + if config.openai_proxy: + openai.proxy = config.openai_proxy def _const_kwargs(self, messages: list[dict]) -> dict: kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3} diff --git a/tests/metagpt/provider/test_zhipuai_api.py b/tests/metagpt/provider/test_zhipuai_api.py index 4684e8887..dc8b63cc3 100644 --- a/tests/metagpt/provider/test_zhipuai_api.py +++ b/tests/metagpt/provider/test_zhipuai_api.py @@ -35,3 +35,10 @@ async def test_zhipuai_acompletion(mocker): assert resp["code"] == 200 assert "chatglm-turbo" in resp["data"]["choices"][0]["content"] + +def test_zhipuai_proxy(mocker): + import openai + from metagpt.config import CONFIG + CONFIG.openai_proxy = 'http://127.0.0.1:8080' + _ = ZhiPuAIGPTAPI() + assert openai.proxy == CONFIG.openai_proxy From bbdbe93809025e821c8f7e9ccaec52ea8bbaa384 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Tue, 26 Dec 2023 19:09:00 +0800 Subject: [PATCH 8/9] fix #560 --- metagpt/roles/researcher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 27f046878..0f342de1c 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -5,6 +5,7 @@ """ import asyncio +import re from pydantic import BaseModel @@ -95,9 +96,11 @@ class Researcher(Role): return msg def write_report(self, topic: str, content: str): + filename = re.sub(r'[\\/:"*?<>|]+', " ", topic) + filename = filename.replace("\n", "") if not RESEARCH_PATH.exists(): RESEARCH_PATH.mkdir(parents=True) - filepath = RESEARCH_PATH / f"{topic}.md" + filepath = RESEARCH_PATH / f"{filename}.md" filepath.write_text(content) From 255f9c4e4ab349978dea2332c9714600f38960b0 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Tue, 26 Dec 2023 19:09:26 +0800 Subject: [PATCH 9/9] add ut for researcher --- metagpt/actions/research.py | 14 ++-- tests/metagpt/actions/test_research.py | 105 +++++++++++++++++++++++++ tests/metagpt/roles/test_researcher.py | 16 ++++ 3 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 tests/metagpt/actions/test_research.py diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index c47a77bdd..5057c3d3a 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -85,7 +85,7 @@ class CollectLinks(Action): llm: BaseGPTAPI = Field(default_factory=LLM) desc: str = "Collect links from a search engine." search_engine: SearchEngine = Field(default_factory=SearchEngine) - rank_func: Union[Callable[[list[str]], None], None] = None + rank_func: Optional[Callable[[list[str]], None]] = None async def run( self, @@ -180,18 +180,18 @@ class WebBrowseAndSummarize(Action): llm: BaseGPTAPI = Field(default_factory=LLM) desc: str = "Explore the web and provide summaries of articles and webpages." browse_func: Union[Callable[[list[str]], None], None] = None - web_browser_engine: WebBrowserEngine = Field( - default_factory=lambda: WebBrowserEngine( - engine=WebBrowserEngineType.CUSTOM if WebBrowseAndSummarize.browse_func else None, - run_func=WebBrowseAndSummarize.browse_func, - ) - ) + web_browser_engine: Optional[WebBrowserEngine] = None def __init__(self, **kwargs): super().__init__(**kwargs) if CONFIG.model_for_researcher_summary: self.llm.model = CONFIG.model_for_researcher_summary + self.web_browser_engine = WebBrowserEngine( + engine=WebBrowserEngineType.CUSTOM if self.browse_func else None, + run_func=self.browse_func, + ) + async def run( self, url: str, diff --git a/tests/metagpt/actions/test_research.py b/tests/metagpt/actions/test_research.py new file mode 100644 index 000000000..bc1982c5d --- /dev/null +++ b/tests/metagpt/actions/test_research.py @@ -0,0 +1,105 @@ +import pytest + +from metagpt.actions import research + + +@pytest.mark.asyncio +async def test_collect_links(mocker): + async def mock_llm_ask(self, prompt: str, system_msgs): + if "Please provide up to 2 necessary keywords" in prompt: + return '["metagpt", "llm"]' + + elif "Provide up to 4 queries related to your research topic" in prompt: + return ( + '["MetaGPT use cases", "The roadmap of MetaGPT", ' + '"The function of MetaGPT", "What llm MetaGPT support"]' + ) + elif "sort the remaining search results" in prompt: + return "[1,2]" + + mocker.patch("metagpt.provider.base_gpt_api.BaseGPTAPI.aask", mock_llm_ask) + resp = await research.CollectLinks().run("The application of MetaGPT") + for i in ["MetaGPT use cases", "The roadmap of MetaGPT", "The function of MetaGPT", "What llm MetaGPT support"]: + assert i in resp + + +@pytest.mark.asyncio +async def test_collect_links_with_rank_func(mocker): + rank_before = [] + rank_after = [] + url_per_query = 4 + + def rank_func(results): + results = results[:url_per_query] + rank_before.append(results) + results = results[::-1] + rank_after.append(results) + return results + + mocker.patch("metagpt.provider.base_gpt_api.BaseGPTAPI.aask", mock_collect_links_llm_ask) + resp = await research.CollectLinks(rank_func=rank_func).run("The application of MetaGPT") + for x, y, z in zip(rank_before, rank_after, resp.values()): + assert x[::-1] == y + assert [i["link"] for i in y] == z + + +@pytest.mark.asyncio +async def test_web_browse_and_summarize(mocker): + async def mock_llm_ask(*args, **kwargs): + return "metagpt" + + mocker.patch("metagpt.provider.base_gpt_api.BaseGPTAPI.aask", mock_llm_ask) + url = "https://github.com/geekan/MetaGPT" + url2 = "https://github.com/trending" + query = "What's new in metagpt" + resp = await research.WebBrowseAndSummarize().run(url, query=query) + + assert len(resp) == 1 + assert url in resp + assert resp[url] == "metagpt" + + resp = await research.WebBrowseAndSummarize().run(url, url2, query=query) + assert len(resp) == 2 + + async def mock_llm_ask(*args, **kwargs): + return "Not relevant." + + mocker.patch("metagpt.provider.base_gpt_api.BaseGPTAPI.aask", mock_llm_ask) + resp = await research.WebBrowseAndSummarize().run(url, query=query) + + assert len(resp) == 1 + assert url in resp + assert resp[url] is None + + +@pytest.mark.asyncio +async def test_conduct_research(mocker): + data = None + + async def mock_llm_ask(*args, **kwargs): + nonlocal data + data = f"# Research Report\n## Introduction\n{args} {kwargs}" + return data + + mocker.patch("metagpt.provider.base_gpt_api.BaseGPTAPI.aask", mock_llm_ask) + content = ( + "MetaGPT takes a one line requirement as input and " + "outputs user stories / competitive analysis / requirements / data structures / APIs / documents, etc." + ) + + resp = await research.ConductResearch().run("The application of MetaGPT", content) + assert resp == data + + +async def mock_collect_links_llm_ask(self, prompt: str, system_msgs): + if "Please provide up to 2 necessary keywords" in prompt: + return '["metagpt", "llm"]' + + elif "Provide up to 4 queries related to your research topic" in prompt: + return ( + '["MetaGPT use cases", "The roadmap of MetaGPT", ' '"The function of MetaGPT", "What llm MetaGPT support"]' + ) + elif "sort the remaining search results" in prompt: + return "[1,2]" + + return "" diff --git a/tests/metagpt/roles/test_researcher.py b/tests/metagpt/roles/test_researcher.py index dd130662d..83e90de66 100644 --- a/tests/metagpt/roles/test_researcher.py +++ b/tests/metagpt/roles/test_researcher.py @@ -32,3 +32,19 @@ async def test_researcher(mocker): researcher.RESEARCH_PATH = Path(dirname) await researcher.Researcher().run(topic) assert (researcher.RESEARCH_PATH / f"{topic}.md").read_text().startswith("# Research Report") + + +def test_write_report(mocker): + with TemporaryDirectory() as dirname: + for i, topic in enumerate( + [ + ("1./metagpt"), + ('2.:"metagpt'), + ("3.*?<>|metagpt"), + ("4. metagpt\n"), + ] + ): + researcher.RESEARCH_PATH = Path(dirname) + content = "# Research Report" + researcher.Researcher().write_report(topic, content) + assert (researcher.RESEARCH_PATH / f"{i+1}. metagpt.md").read_text().startswith("# Research Report")