From f0edea65feb351b02110c5f9bde9a7f1eaee6781 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 2 Aug 2024 12:26:08 +0800 Subject: [PATCH 1/8] fixbug: #1430 --- metagpt/actions/project_management.py | 2 +- metagpt/actions/project_management_an.py | 14 +++++++------- metagpt/actions/write_code_an_draft.py | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 67a614d6f..aede61a2e 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -84,7 +84,7 @@ class WriteTasks(Action): async def _update_requirements(self, doc): m = json.loads(doc.content) - packages = set(m.get("Required Python packages", set())) + packages = set(m.get("Required packages", set())) requirement_doc = await self.repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME) if not requirement_doc: requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="") diff --git a/metagpt/actions/project_management_an.py b/metagpt/actions/project_management_an.py index 0417c0ce4..f53062433 100644 --- a/metagpt/actions/project_management_an.py +++ b/metagpt/actions/project_management_an.py @@ -5,14 +5,14 @@ @Author : alexanderwu @File : project_management_an.py """ -from typing import List +from typing import List, Optional from metagpt.actions.action_node import ActionNode -REQUIRED_PYTHON_PACKAGES = ActionNode( - key="Required Python packages", - expected_type=List[str], - instruction="Provide required Python packages in requirements.txt format.", +REQUIRED_PACKAGES = ActionNode( + key="Required packages", + expected_type=Optional[List[str]], + instruction="Provide required third-party packages in requirements.txt format.", example=["flask==1.1.2", "bcrypt==3.2.0"], ) @@ -97,7 +97,7 @@ ANYTHING_UNCLEAR_PM = ActionNode( ) NODES = [ - REQUIRED_PYTHON_PACKAGES, + REQUIRED_PACKAGES, REQUIRED_OTHER_LANGUAGE_PACKAGES, LOGIC_ANALYSIS, TASK_LIST, @@ -107,7 +107,7 @@ NODES = [ ] REFINED_NODES = [ - REQUIRED_PYTHON_PACKAGES, + REQUIRED_PACKAGES, REQUIRED_OTHER_LANGUAGE_PACKAGES, REFINED_LOGIC_ANALYSIS, REFINED_TASK_LIST, diff --git a/metagpt/actions/write_code_an_draft.py b/metagpt/actions/write_code_an_draft.py index ce030b0e9..20ed201a3 100644 --- a/metagpt/actions/write_code_an_draft.py +++ b/metagpt/actions/write_code_an_draft.py @@ -139,7 +139,7 @@ Language: Please use the same language as the user requirement, but the title an end", "Anything UNCLEAR": "目前项目要求明确，没有不清楚的地方。"} ## Tasks -{"Required Python packages": ["无需Python包"], "Required Other language third-party packages": ["vue.js"], "Logic Analysis": [["index.html", "作为游戏的入口文件和主要的HTML结构"], ["styles.css", "包含所有的CSS样式，确保游戏界面美观"], ["main.js", "包含Main类，负责初始化游戏和绑定事件"], ["game.js", "包含Game类，负责游戏逻辑，如开始游戏、移动方块等"], ["storage.js", "包含Storage类，用于获取和设置玩家的最高分"]], "Task list": ["index.html", "styles.css", "storage.js", "game.js", "main.js"], "Full API spec": "", "Shared Knowledge": "\'game.js\' 包含游戏逻辑相关的函数，被 \'main.js\' 调用。", "Anything UNCLEAR": "目前项目要求明确，没有不清楚的地方。"} +{"Required packages": ["无需第三方包"], "Required Other language third-party packages": ["vue.js"], "Logic Analysis": [["index.html", "作为游戏的入口文件和主要的HTML结构"], ["styles.css", "包含所有的CSS样式，确保游戏界面美观"], ["main.js", "包含Main类，负责初始化游戏和绑定事件"], ["game.js", "包含Game类，负责游戏逻辑，如开始游戏、移动方块等"], ["storage.js", "包含Storage类，用于获取和设置玩家的最高分"]], "Task list": ["index.html", "styles.css", "storage.js", "game.js", "main.js"], "Full API spec": "", "Shared Knowledge": "\'game.js\' 包含游戏逻辑相关的函数，被 \'main.js\' 调用。", "Anything UNCLEAR": "目前项目要求明确，没有不清楚的地方。"} ## Code Files ----- index.html From f651e45f537d3a35c7a4c88100f5bba87084a9c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 2 Aug 2024 13:56:10 +0800 Subject: [PATCH 2/8] feat: fix unit test --- tests/metagpt/test_config.py | 4 ++-- tests/metagpt/test_context.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/metagpt/test_config.py b/tests/metagpt/test_config.py index 7ce5765cf..797daf5dc 100644 --- a/tests/metagpt/test_config.py +++ b/tests/metagpt/test_config.py @@ -14,8 +14,8 @@ from tests.metagpt.provider.mock_llm_config import mock_llm_config def test_config_1(): cfg = Config.default() llm = cfg.get_openai_llm() - assert llm is not None - assert llm.api_type == LLMType.OPENAI + if cfg.llm.api_type == LLMType.OPENAI: + assert llm is not None def test_config_from_dict(): diff --git a/tests/metagpt/test_context.py b/tests/metagpt/test_context.py index f8218c44d..a6daf95cd 100644 --- a/tests/metagpt/test_context.py +++ b/tests/metagpt/test_context.py @@ -53,8 +53,8 @@ def test_context_1(): def test_context_2(): ctx = Context() llm = ctx.config.get_openai_llm() - assert llm is not None - assert llm.api_type == LLMType.OPENAI + if ctx.config.llm.api_type == LLMType.OPENAI: + assert llm is not None kwargs = ctx.kwargs assert kwargs is not None From 0592b1f9e1e63b5fda9fc79d419ea60b834a84cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 2 Aug 2024 14:11:28 +0800 Subject: [PATCH 3/8] feat: + requirements --- requirements.txt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 73ca0f87b..ac3b353b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -69,4 +69,6 @@ imap_tools==1.5.0 # Used by metagpt/tools/libs/email_login.py qianfan==0.3.2 dashscope==1.14.1 rank-bm25==0.2.2 # for tool recommendation -jieba==0.42.1 # for tool recommendation \ No newline at end of file +jieba==0.42.1 # for tool recommendation +# llama-index-vector-stores-elasticsearch~=0.2.5 # Used by `metagpt/memory/longterm_memory.py` +# llama-index-vector-stores-chroma~=0.1.10 # Used by `metagpt/memory/longterm_memory.py` \ No newline at end of file From 61537c5d70d8fbf16794a1add5c4aab79f6f4e44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 2 Aug 2024 15:39:38 +0800 Subject: [PATCH 4/8] feat: +gpt 4o tiktoken --- metagpt/utils/token_counter.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 0ba2daa89..a99d351f7 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -32,6 +32,8 @@ TOKEN_COSTS = { "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "gpt-4-vision-preview": {"prompt": 0.01, "completion": 0.03}, # TODO add extra image price calculator "gpt-4-1106-vision-preview": {"prompt": 0.01, "completion": 0.03}, + "gpt-4o": {"prompt": 0.005, "completion": 0.015}, + "gpt-4o-mini": {"prompt": 0.00015, "completion": 0.0006}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, "glm-3-turbo": {"prompt": 0.0007, "completion": 0.0007}, # 128k version, prompt + completion tokens=0.005￥/k-tokens "glm-4": {"prompt": 0.014, "completion": 0.014}, # 128k version, prompt + completion tokens=0.1￥/k-tokens @@ -154,6 +156,8 @@ TOKEN_MAX = { "gpt-4-0613": 8192, "gpt-4-32k": 32768, "gpt-4-32k-0613": 32768, + "gpt-4o-mini": 128000, + "gpt-4o": 128000, "gpt-3.5-turbo-0125": 16385, "gpt-3.5-turbo": 16385, "gpt-3.5-turbo-1106": 16385, @@ -207,6 +211,8 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0125"): "gpt-4-1106-preview", "gpt-4-vision-preview", "gpt-4-1106-vision-preview", + "gpt-4o", + "gpt-4o-mini", }: tokens_per_message = 3 # # every reply is primed with <|start|>assistant<|message|> tokens_per_name = 1 From e199c6b476bca1de6ca35c63af3c082eb91650f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 6 Aug 2024 15:03:24 +0800 Subject: [PATCH 5/8] fixbug: Optional not working --- metagpt/actions/action_node.py | 18 ++++++++++++++++-- tests/metagpt/actions/test_action_node.py | 18 +++++++++++++++--- 2 files changed, 31 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 31e4cc0fc..4c07ed99d 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -236,13 +236,27 @@ class ActionNode: def create_model_class(cls, class_name: str, mapping: Dict[str, Tuple[Type, Any]]): """基于pydantic v2的模型动态生成，用来检验结果类型正确性""" + def is_optional_type(tp): + if typing.get_origin(tp) is Union: + args = typing.get_args(tp) + non_none_types = [arg for arg in args if arg is not type(None)] + return len(non_none_types) == 1 and len(args) == 2 + return False + def check_fields(cls, values): - required_fields = set(mapping.keys()) + all_fields = set(mapping.keys()) + required_fields = set() + for k, v in mapping.items(): + type_v, field_info = v + if is_optional_type(type_v): + continue + required_fields.add(k) + missing_fields = required_fields - set(values.keys()) if missing_fields: raise ValueError(f"Missing fields: {missing_fields}") - unrecognized_fields = set(values.keys()) - required_fields + unrecognized_fields = set(values.keys()) - all_fields if unrecognized_fields: logger.warning(f"Unrecognized fields: {unrecognized_fields}") return values diff --git a/tests/metagpt/actions/test_action_node.py b/tests/metagpt/actions/test_action_node.py index 989e2249c..338d87242 100644 --- a/tests/metagpt/actions/test_action_node.py +++ b/tests/metagpt/actions/test_action_node.py @@ -6,7 +6,7 @@ @File : test_action_node.py """ from pathlib import Path -from typing import List, Tuple +from typing import List, Optional, Tuple import pytest from pydantic import BaseModel, Field, ValidationError @@ -302,6 +302,18 @@ def test_action_node_from_pydantic_and_print_everything(): assert "tasks" in code, "tasks should be in code" +def test_optional(): + mapping = { + "Logic Analysis": (Optional[List[Tuple[str, str]]], Field(default=None)), + "Task list": (Optional[List[str]], None), + "Anything UNCLEAR": (Optional[str], None), + } + m = {"Anything UNCLEAR": "a"} + t = ActionNode.create_model_class("test_class_1", mapping) + + t1 = t(**m) + assert t1 + + if __name__ == "__main__": - test_create_model_class() - test_create_model_class_with_mapping() + pytest.main([__file__, "-s"]) From 284699224ec70b8abf85ffb8971db33cf2c7b77f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 6 Aug 2024 15:29:10 +0800 Subject: [PATCH 6/8] fixbug: Optional not working --- metagpt/actions/action_node.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/action_node.py b/metagpt/actions/action_node.py index 4c07ed99d..f0ffbe085 100644 --- a/metagpt/actions/action_node.py +++ b/metagpt/actions/action_node.py @@ -236,19 +236,12 @@ class ActionNode: def create_model_class(cls, class_name: str, mapping: Dict[str, Tuple[Type, Any]]): """基于pydantic v2的模型动态生成，用来检验结果类型正确性""" - def is_optional_type(tp): - if typing.get_origin(tp) is Union: - args = typing.get_args(tp) - non_none_types = [arg for arg in args if arg is not type(None)] - return len(non_none_types) == 1 and len(args) == 2 - return False - def check_fields(cls, values): all_fields = set(mapping.keys()) required_fields = set() for k, v in mapping.items(): type_v, field_info = v - if is_optional_type(type_v): + if ActionNode.is_optional_type(type_v): continue required_fields.add(k) @@ -732,3 +725,12 @@ class ActionNode: root_node.add_child(child_node) return root_node + + @staticmethod + def is_optional_type(tp) -> bool: + """Return True if `tp` is `typing.Optional[...]`""" + if typing.get_origin(tp) is Union: + args = typing.get_args(tp) + non_none_types = [arg for arg in args if arg is not type(None)] + return len(non_none_types) == 1 and len(args) == 2 + return False From 9086cccc49adebed7ba5ebcd2e76660c7a9e9763 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 6 Aug 2024 15:33:25 +0800 Subject: [PATCH 7/8] fixbug: Optional not working --- tests/metagpt/actions/test_action_node.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/metagpt/actions/test_action_node.py b/tests/metagpt/actions/test_action_node.py index 338d87242..58a6dd517 100644 --- a/tests/metagpt/actions/test_action_node.py +++ b/tests/metagpt/actions/test_action_node.py @@ -306,6 +306,7 @@ def test_optional(): mapping = { "Logic Analysis": (Optional[List[Tuple[str, str]]], Field(default=None)), "Task list": (Optional[List[str]], None), + "Plan": (Optional[str], ""), "Anything UNCLEAR": (Optional[str], None), } m = {"Anything UNCLEAR": "a"} From 534e231195e5c98d039307a701c21212f2623b93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 6 Aug 2024 16:09:37 +0800 Subject: [PATCH 8/8] feat: merge main --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cfac78aa1..42483f5fe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ openai>=1.6.1 openpyxl beautifulsoup4==4.12.3 pandas==2.1.1 -pydantic==2.6.4 +pydantic>=2.5.3 #pygame==2.1.3 #pymilvus==2.2.8 # pytest==7.2.2 # test extras require