diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index 87c5b3c0d..91b5af49e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -8,6 +8,7 @@ from enum import Enum from metagpt.actions.action import Action +from metagpt.actions.action_output import ActionOutput from metagpt.actions.write_prd import WritePRD from metagpt.actions.write_prd_review import WritePRDReview diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index e28f56e40..a390a8350 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -9,6 +9,10 @@ from typing import Optional from abc import ABC from metagpt.llm import LLM +from metagpt.actions.action_output import ActionOutput +from tenacity import retry, stop_after_attempt, wait_fixed +from pydantic import BaseModel +from metagpt.utils.common import OutputParser class Action(ABC): @@ -21,6 +25,8 @@ class Action(ABC): self.prefix = "" self.profile = "" self.desc = "" + self.content = "" + self.instruct_content = None def set_prefix(self, prefix, profile): """Set prefix for later usage""" @@ -40,6 +46,20 @@ class Action(ABC): system_msgs.append(self.prefix) return await self.llm.aask(prompt, system_msgs) + @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + async def _aask_v1(self, prompt: str, output_class_name: str, + output_data_mapping: dict, + system_msgs: Optional[list[str]] = None) -> ActionOutput: + """Append default prefix""" + if not system_msgs: + system_msgs = [] + system_msgs.append(self.prefix) + content = await self.llm.aask(prompt, system_msgs) + output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping) + parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping) + instruct_content = output_class(**parsed_data) + return ActionOutput(content, instruct_content) + async def run(self, *args, **kwargs): """Run action""" raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index fd7603834..a8f6a6ceb 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,7 +7,9 @@ """ import shutil from pathlib import Path +from typing import List, Tuple +from metagpt.actions import ActionOutput from metagpt.actions import Action from metagpt.const import WORKSPACE_ROOT from metagpt.utils.common import CodeParser @@ -18,6 +20,9 @@ from metagpt.utils.mermaid import mermaid_to_file PROMPT_TEMPLATE = """ # Context {context} + +## Format example +{format_example} ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately @@ -37,6 +42,53 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. """ +FORMAT_EXAMPLE = """ +--- +## Implementation approach +We will ... + +## Python package name +```python +"snake_game" +``` + +## File list +```python +[ + "main.py", +] +``` + +## Data structures and interface definitions +```mermaid +classDiagram + class Game{ + +int score + } + ... + Game "1" -- "1" Food: has +``` + +## Program call flow +```mermaid +sequenceDiagram + participant M as Main + ... + G->>M: end game +``` + +## Anything UNCLEAR +The requirement is clear to me. +--- +""" +OUTPUT_MAPPING = { + "Implementation approach": (str, ...), + "Python package name": (str, ...), + "File list": (List[str], ...), + "Data structures and interface definitions": (str, ...), + "Program call flow": (str, ...), + "Anything UNCLEAR": (str, ...), +} class WriteDesign(Action): @@ -60,17 +112,22 @@ class WriteDesign(Action): logger.info(f"Saving PRD to {prd_file}") prd_file.write_text(prd) - def _save_system_design(self, docs_path, resources_path, system_design): - data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=system_design) - seq_flow = CodeParser.parse_code(block="Program call flow", text=system_design) + def _save_system_design(self, docs_path, resources_path, content): + data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) + seq_flow = CodeParser.parse_code(block="Program call flow", text=content) mermaid_to_file(data_api_design, resources_path / 'data_api_design') mermaid_to_file(seq_flow, resources_path / 'seq_flow') system_design_file = docs_path / 'system_design.md' logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text(system_design) + system_design_file.write_text(content) def _save(self, context, system_design): - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) + if isinstance(system_design, ActionOutput): + content = system_design.content + ws_name = CodeParser.parse_str(block="Python package name", text=content) + else: + content = system_design + ws_name = CodeParser.parse_str(block="Python package name", text=system_design) workspace = WORKSPACE_ROOT / ws_name self.recreate_workspace(workspace) docs_path = workspace / 'docs' @@ -78,10 +135,11 @@ class WriteDesign(Action): docs_path.mkdir(parents=True, exist_ok=True) resources_path.mkdir(parents=True, exist_ok=True) self._save_prd(docs_path, resources_path, context[-1].content) - self._save_system_design(docs_path, resources_path, system_design) + self._save_system_design(docs_path, resources_path, content) async def run(self, context): - prompt = PROMPT_TEMPLATE.format(context=context) - system_design = await self._aask(prompt) + prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) + # system_design = await self._aask(prompt) + system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) self._save(context, system_design) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index c93463849..1dc18dc90 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -5,15 +5,21 @@ @Author : alexanderwu @File : project_management.py """ +from typing import List, Tuple from metagpt.actions.action import Action +from metagpt.actions.action_output import ActionOutput from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.utils.common import CodeParser +from metagpt.utils.common import OutputParser, CodeParser +from tenacity import retry, stop_after_attempt, wait_fixed -PROMPT_TEMPLATE = """ +PROMPT_TEMPLATE = ''' # Context {context} + +## Format example +{format_example} ----- Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them @@ -33,10 +39,72 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +''' + +FORMAT_EXAMPLE = ''' +--- +## Required Python third-party packages +```python """ +flask==1.1.2 +""" +``` + +## Required Other language third-party packages +```python +""" +No third-party ... +""" +``` + +## Full API spec +```python +""" +openapi: 3.0.0 +... +description: A JSON object ... +""" +``` + +## Logic Analysis +```python +[ + ("game.py", "Contains ..."), +] +``` + +## Task list +```python +[ + "game.py", +] +``` + +## Shared Knowledge +```python +""" +'game.py' contains ... +""" +``` + +## Anything UNCLEAR +We need ... how to start. +--- +''' + +OUTPUT_MAPPING = { + "Required Python third-party packages": (str, ...), + "Required Other language third-party packages": (str, ...), + "Full API spec": (str, ...), + "Logic Analysis": (List[Tuple[str, str]], ...), + "Task list": (List[str], ...), + "Shared Knowledge": (str, ...), + "Anything UNCLEAR": (str, ...), +} class WriteTasks(Action): + def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) @@ -46,9 +114,8 @@ class WriteTasks(Action): file_path.write_text(rsp) async def run(self, context): - prompt = PROMPT_TEMPLATE.format(context=context) - rsp = await self._aask(prompt) - self._save(context, rsp) + prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) + rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 3f413d5fb..de06baaf5 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -31,6 +31,9 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W """ ## {filename}: Please encapsulate your code within triple quotes. Focus your efforts on implementing ONLY WITHIN THIS FILE. Any class or function labeled as MISSING-DESIGN should be implemented IN THIS FILE ALONE. Do NOT make changes to any other files. +OUTPUT_MAPPING = { + "{filename}": (str, ...), +} class WriteCode(Action): @@ -47,6 +50,7 @@ class WriteCode(Action): return design = [i for i in context if i.cause_by == WriteDesign][0] + ws_name = CodeParser.parse_str(block="Python package name", text=design.content) ws_path = WORKSPACE_ROOT / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): @@ -63,5 +67,6 @@ class WriteCode(Action): context = kwargs['context'] logger.info(f'Writing {filename}..') code_rsp = await self._aask(prompt) + # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) self._save(context, filename, code_rsp) return code_rsp diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 4093ab018..993e28ecf 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -5,10 +5,11 @@ @Author : alexanderwu @File : write_prd.py """ -from metagpt.actions import Action +from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SEARCH_AND_SUMMARIZE_SYSTEM, SearchAndSummarize, \ SEARCH_AND_SUMMARIZE_PROMPT, SEARCH_AND_SUMMARIZE_SYSTEM_EN_US from metagpt.logs import logger +from typing import List, Tuple PROMPT_TEMPLATE = """ # Context @@ -36,10 +37,13 @@ quadrantChart "Campaign F": [0.35, 0.78] "Our Target Product": [0.5, 0.6] ``` + +## Format example +{format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. ## Original Requirements: Provide as Plain text, place the polished complete original requirements here @@ -56,15 +60,72 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD W ## Requirement Pool: Provided as Python list[str, str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower ## Anything UNCLEAR: Provide as Plain text. Make clear here. - """ +FORMAT_EXAMPLE = """ +--- +## Original Requirements +The boss ... + +## Product Goals +```python +[ + "Create a ...", +] +``` + +## User Stories +```python +[ + "As a user, ...", +] +``` + +## Competitive Analysis +```python +[ + "Python Snake Game: ...", +] +``` + +## Competitive Quadrant Chart +```mermaid +quadrantChart + title Reach and engagement of campaigns + ... + "Our Target Product": [0.6, 0.7] +``` + +## Requirement Analysis +The product should be a ... + +## Requirement Pool +```python +[ + ("End game ...", "P0") +] +``` + +## Anything UNCLEAR +There are no unclear points. +--- +""" +OUTPUT_MAPPING = { + "Original Requirements": (str, ...), + "Product Goals": (List[str], ...), + "User Stories": (List[str], ...), + "Competitive Analysis": (List[str], ...), + "Competitive Quadrant Chart": (str, ...), + "Requirement Analysis": (str, ...), + "Requirement Pool": (List[Tuple[str, str]], ...), + "Anything UNCLEAR": (str, ...), +} class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, requirements, *args, **kwargs) -> str: + async def run(self, requirements, *args, **kwargs) -> ActionOutput: sas = SearchAndSummarize() rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" @@ -72,6 +133,7 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) - prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info) - prd = await self._aask(prompt) + prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info, + format_example=FORMAT_EXAMPLE) + prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) return prd diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e0ffa72cc..89e58c201 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -173,7 +173,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "max_tokens": CONFIG.max_tokens_rsp, "n": 1, "stop": None, - "temperature": 0.5 + "temperature": 0.3 } else: kwargs = { @@ -182,7 +182,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "max_tokens": CONFIG.max_tokens_rsp, "n": 1, "stop": None, - "temperature": 0.5 + "temperature": 0.3 } return kwargs diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 87fefc20f..c8cebf680 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -22,7 +22,7 @@ from collections import OrderedDict async def gather_ordered_k(coros, k) -> list: tasks = OrderedDict() - results = [None]*len(coros) + results = [None] * len(coros) done_queue = asyncio.Queue() for i, coro in enumerate(coros): @@ -59,6 +59,8 @@ class Engineer(Role): @classmethod def parse_tasks(self, task_msg: Message) -> list[str]: + if not task_msg.instruct_content: + return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod @@ -67,6 +69,8 @@ class Engineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: + if not system_design_msg.instruct_content: + return system_design_msg.instruct_content.dict().get("Python package name") return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 7777ba79c..7aae5cd62 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -12,7 +12,7 @@ from typing import Type, Iterable from metagpt.logs import logger # from metagpt.environment import Environment -from metagpt.actions import Action +from metagpt.actions import Action, ActionOutput from metagpt.llm import LLM from metagpt.schema import Message from metagpt.memory import Memory @@ -45,7 +45,6 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi """ - @dataclass class RoleSetting: """角色设定""" @@ -83,6 +82,7 @@ class RoleContext: class Role: """角色/代理""" + def __init__(self, name="", profile="", goal="", constraints="", desc=""): self._llm = LLM() self._setting = RoleSetting(name, profile, goal, constraints, desc) @@ -153,7 +153,11 @@ class Role: logger.info(f"{self._setting}: ready to {self._rc.todo}") response = await self._rc.todo.run(self._rc.important_memory) # logger.info(response) - msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + if isinstance(response, ActionOutput): + msg = Message(content=response.content, instruct_content=response.instruct_content, + role=self.profile, cause_by=type(self._rc.todo)) + else: + msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) # logger.debug(f"{response}") diff --git a/metagpt/schema.py b/metagpt/schema.py index 9df25be60..31b579035 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,7 +10,7 @@ from dataclasses import dataclass, field from typing import Type, TypedDict from metagpt.logs import logger -# from pydantic import BaseModel +from pydantic import BaseModel @@ -23,6 +23,7 @@ class RawMessage(TypedDict): class Message: """list[: ]""" content: str + instruct_content: BaseModel = field(default=None) role: str = field(default='user') # system / user / assistant cause_by: Type["Action"] = field(default="") diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 8e3a49af9..f74d2d8b7 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -10,7 +10,7 @@ import ast import inspect import re -from typing import Union +from typing import Union, List, Tuple from metagpt.logs import logger from langchain.schema import AgentAction, AgentFinish, OutputParserException @@ -27,6 +27,112 @@ def check_cmd_exists(command) -> int: return result +class OutputParser: + + @classmethod + def parse_blocks(cls, text: str): + # 首先根据"##"将文本分割成不同的block + blocks = text.split("##") + + # 创建一个字典，用于存储每个block的标题和内容 + block_dict = {} + + # 遍历所有的block + for block in blocks: + # 如果block不为空，则继续处理 + if block.strip() != "": + # 将block的标题和内容分开，并分别去掉前后的空白字符 + block_title, block_content = block.split("\n", 1) + # LLM可能出错，在这里做一下修正 + if block_title[-1] == ":": + block_title = block_title[:-1] + block_dict[block_title.strip()] = block_content.strip() + + return block_dict + + @classmethod + def parse_code(cls, text: str, lang: str = "") -> str: + pattern = rf'```{lang}.*?\s+(.*?)```' + match = re.search(pattern, text, re.DOTALL) + if match: + code = match.group(1) + else: + raise Exception + return code + + @classmethod + def parse_str(cls, text: str): + text = text.split("=")[-1] + text = text.strip().strip("'").strip("\"") + return text + + @classmethod + def parse_file_list(cls, text: str) -> list[str]: + # Regular expression pattern to find the tasks list. + pattern = r'\s*(.*=.*)?(\[.*\])' + + # Extract tasks list string using regex. + match = re.search(pattern, text, re.DOTALL) + if match: + tasks_list_str = match.group(2) + + # Convert string representation of list to a Python list using ast.literal_eval. + tasks = ast.literal_eval(tasks_list_str) + else: + raise Exception + return tasks + + @classmethod + def parse_data(cls, data): + block_dict = cls.parse_blocks(data) + parsed_data = {} + for block, content in block_dict.items(): + # 尝试去除code标记 + try: + content = cls.parse_code(text=content) + except Exception: + pass + + # 尝试解析list + try: + content = cls.parse_file_list(text=content) + except Exception: + pass + parsed_data[block] = content + return parsed_data + + @classmethod + def parse_data_with_mapping(cls, data, mapping): + block_dict = cls.parse_blocks(data) + parsed_data = {} + for block, content in block_dict.items(): + # 尝试去除code标记 + try: + content = cls.parse_code(text=content) + except Exception: + pass + typing_define = mapping.get(block, None) + if isinstance(typing_define, tuple): + typing = typing_define[0] + else: + typing = typing_define + if typing == List[str] or typing == List[Tuple[str, str]]: + # 尝试解析list + try: + content = cls.parse_file_list(text=content) + except Exception: + pass + # TODO: 多余的引号去除有风险，后期再解决 + # elif typing == str: + # # 尝试去除多余的引号 + # try: + # content = cls.parse_str(text=content) + # except Exception: + # pass + parsed_data[block] = content + return parsed_data + + class CodeParser: @classmethod @@ -56,7 +162,7 @@ class CodeParser: return block_dict @classmethod - def parse_code(cls, block: str, text: str, lang: str="") -> str: + def parse_code(cls, block: str, text: str, lang: str = "") -> str: if block: text = cls.parse_block(block, text) pattern = rf'```{lang}.*?\s+(.*?)```' @@ -70,16 +176,17 @@ class CodeParser: return code @classmethod - def parse_str(cls, block: str, text: str, lang: str=""): + def parse_str(cls, block: str, text: str, lang: str = ""): code = cls.parse_code(block, text, lang) code = code.split("=")[-1] code = code.strip().strip("'").strip("\"") return code @classmethod - def parse_file_list(cls, block: str, text: str, lang: str="") -> list[str]: + def parse_file_list(cls, block: str, text: str, lang: str = "") -> list[str]: # Regular expression pattern to find the tasks list. code = cls.parse_code(block, text, lang) + print(code) pattern = r'\s*(.*=.*)?(\[.*\])' # Extract tasks list string using regex. @@ -96,6 +203,7 @@ class CodeParser: class NoMoneyException(Exception): """Raised when the operation cannot be completed due to insufficient funds""" + def __init__(self, amount, message="Insufficient funds"): self.amount = amount self.message = message @@ -154,4 +262,4 @@ if __name__ == '__main__': logger.info(rsp) rsp = parser.parse(final_answer_sample) - logger.info(rsp) \ No newline at end of file + logger.info(rsp)