diff --git a/config/config.yaml b/config/config.yaml index bed67083c..9acdbe8a1 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -7,9 +7,9 @@ ## Or, you can configure OPENAI_PROXY to access official OPENAI_API_BASE. OPENAI_API_BASE: "https://api.openai.com/v1" #OPENAI_PROXY: "http://127.0.0.1:8118" -#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model -OPENAI_API_MODEL: "gpt-4" -MAX_TOKENS: 1500 +#OPENAI_API_KEY: "YOUR_API_KEY" # set the value to sk-xxx if you host the openai interface for open llm model +OPENAI_API_MODEL: "gpt-4-1106-preview" +MAX_TOKENS: 4096 RPM: 10 #### if Spark diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 325e7c260..bcb9c0c1d 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -5,13 +5,14 @@ Author: garylin2099 ''' import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.const import METAGPT_ROOT +from metagpt.config import CONFIG from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: +with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() @@ -49,7 +50,7 @@ class CreateAgent(Action): pattern = r'```python(.*)```' match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" - with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: + with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text diff --git a/examples/debate.py b/examples/debate.py index a37e60848..e62a5aaa1 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -8,7 +8,7 @@ import platform import fire from metagpt.team import Team -from metagpt.actions import Action, BossRequirement +from metagpt.actions import Action, UserRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger @@ -49,7 +49,7 @@ class Debator(Role): ): super().__init__(name, profile, **kwargs) self._init_actions([SpeakAloud]) - self._watch([BossRequirement, SpeakAloud]) + self._watch([UserRequirement, SpeakAloud]) self.name = name self.opponent_name = opponent_name @@ -88,7 +88,7 @@ async def debate(idea: str, investment: float = 3.0, n_round: int = 5): team = Team() team.hire([Biden, Trump]) team.invest(investment) - team.start_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first + team.run_project(idea, send_to="Biden") # send debate topic to Biden and let him speak first await team.run(n_round=n_round) diff --git a/examples/sk_agent.py b/examples/sk_agent.py index a7513e838..647ea4380 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -13,7 +13,7 @@ from semantic_kernel.planning import SequentialPlanner # from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -39,7 +39,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def sequential_planner_example(): @@ -53,7 +53,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def basic_planner_web_search_example(): @@ -64,7 +64,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def action_planner_example(): @@ -75,7 +75,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=UserRequirement)) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index b004bd58e..79ff94b3e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -9,7 +9,7 @@ from enum import Enum from metagpt.actions.action import Action from metagpt.actions.action_output import ActionOutput -from metagpt.actions.add_requirement import BossRequirement +from metagpt.actions.add_requirement import UserRequirement from metagpt.actions.debug_error import DebugError from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview @@ -28,7 +28,7 @@ from metagpt.actions.write_test import WriteTest class ActionType(Enum): """All types of Actions, used for indexing.""" - ADD_REQUIREMENT = BossRequirement + ADD_REQUIREMENT = UserRequirement WRITE_PRD = WritePRD WRITE_PRD_REVIEW = WritePRDReview WRITE_DESIGN = WriteDesign diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 790295d55..f8016b8a2 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -30,6 +30,10 @@ class Action(ABC): self.desc = "" self.content = "" self.instruct_content = None + self.env = None + + def set_env(self, env): + self.env = env def set_prefix(self, prefix, profile): """Set prefix for later usage""" diff --git a/metagpt/actions/add_requirement.py b/metagpt/actions/add_requirement.py index 7dc09d062..8e2c56a62 100644 --- a/metagpt/actions/add_requirement.py +++ b/metagpt/actions/add_requirement.py @@ -8,7 +8,7 @@ from metagpt.actions import Action -class BossRequirement(Action): - """Boss Requirement without any implementation details""" +class UserRequirement(Action): + """User Requirement without any implementation details""" async def run(self, *args, **kwargs): raise NotImplementedError diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..9e2bfc12c 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,11 +11,9 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -27,21 +25,21 @@ templates = { ## Format example {format_example} ----- -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -50,9 +48,9 @@ and only output the json inside this tag, nothing else [CONTENT] { "Implementation approach": "We will ...", - "Python package name": "snake_game", + "project_name": "snake_game", "File list": ["main.py"], - "Data structures and interface definitions": ' + "Data structures and interfaces": ' classDiagram class Game{ +int score @@ -80,21 +78,21 @@ and only output the json inside this tag, nothing else {format_example} ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Max Output: 8192 chars or 2048 tokens. Try to use them up. -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ @@ -102,7 +100,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Implementation approach We will ... -## Python package name +## project_name ```python "snake_game" ``` @@ -114,7 +112,7 @@ We will ... ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Game{ @@ -141,9 +139,9 @@ The requirement is clear to me. OUTPUT_MAPPING = { "Implementation approach": (str, ...), - "Python package name": (str, ...), + "project_name": (str, ...), "File list": (List[str], ...), - "Data structures and interface definitions": (str, ...), + "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -173,12 +171,12 @@ class WriteDesign(Action): if context[-1].instruct_content: logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) + prd_file.write_text(context[-1].instruct_content.json(ensure_ascii=False), encoding='utf-8') async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) + "Data structures and interfaces" + ] # CodeParser.parse_code(block="Data structures and interfaces", text=content) seq_flow = system_design.instruct_content.dict()[ "Program call flow" ] # CodeParser.parse_code(block="Program call flow", text=content) @@ -186,14 +184,14 @@ class WriteDesign(Action): await mermaid_to_file(seq_flow, resources_path / "seq_flow") system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) + system_design_file.write_text(system_design.instruct_content.json(ensure_ascii=False), encoding='utf-8') async def _save(self, context, system_design): if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] + project_name = system_design.instruct_content.dict()["project_name"] else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name + project_name = CodeParser.parse_str(block="project_name", text=system_design) + workspace = CONFIG.workspace_path / project_name self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" @@ -207,11 +205,11 @@ class WriteDesign(Action): prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr - setattr( - system_design.instruct_content, - "Python package name", - system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), - ) + # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr + # setattr( + # system_design.instruct_content, + # "project_name", + # system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), + # ) await self._save(context, system_design) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..805226a25 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -9,10 +9,8 @@ from typing import List from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown templates = { "json": { @@ -24,22 +22,23 @@ templates = { {format_example} ----- Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them -Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +ATTENTION: Output carefully referenced "Format example" in format. -## Required Python third-party packages: Provided in requirements.txt format +## Required Python third-party packages: Provide Python list[str] in requirements.txt format -## Required Other language third-party packages: Provided in requirements.txt format - -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. +## Required Other language third-party packages: Provide Python list[str] in requirements.txt format ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -53,17 +52,17 @@ and only output the json inside this tag, nothing else "Required Other language third-party packages": [ "No third-party ..." ], + "Logic Analysis": [ + ["game.py", "Contains..."] + ], + "Task list": [ + "game.py" + ], "Full API spec": """ openapi: 3.0.0 ... description: A JSON object ... """, - "Logic Analysis": [ - ["game.py","Contains..."] - ], - "Task list": [ - "game.py" - ], "Shared Knowledge": """ 'game.py' contains ... """, @@ -87,15 +86,15 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Required Other language third-party packages: Provided in requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. - ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. """, "FORMAT_EXAMPLE": ''' @@ -127,14 +126,16 @@ description: A JSON object ... ## Logic Analysis ```python [ - ["game.py", "Contains ..."], + ["index.js", "Contains ..."], + ["main.py", "Contains ..."], ] ``` ## Task list ```python [ - "game.py", + "index.js", + "main.py", ] ``` @@ -168,14 +169,14 @@ class WriteTasks(Action): def _save(self, context, rsp): if context[-1].instruct_content: - ws_name = context[-1].instruct_content.dict()["Python package name"] + ws_name = context[-1].instruct_content.dict()["project_name"] else: - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" - file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) + ws_name = CodeParser.parse_str(block="project_name", text=context[-1].content) + file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" + file_path.write_text(rsp.instruct_content.json(ensure_ascii=False)) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" + requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) async def run(self, context, format=CONFIG.prompt_format): diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py new file mode 100644 index 000000000..a85d3cdeb --- /dev/null +++ b/metagpt/actions/summarize_code.py @@ -0,0 +1,92 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Author : alexanderwu +@File : summarize_code.py +""" + +from tenacity import retry, stop_after_attempt, wait_fixed +from metagpt.actions.action import Action +from metagpt.logs import logger +from metagpt.schema import Message + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to review the code. +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +----- +# Context +{context} +----- + +## Code Review All: 请你对历史所有文件进行阅读，在文件中找到可能的bug，如函数未实现、调用错误、未引用等 + +## Call flow: mermaid代码，根据实现的函数，使用mermaid绘制完整的调用链 + +## Summary: 根据历史文件的实现情况进行总结 + +## TODOs: Python dict[str, str]，这里写出需要修改的文件列表与理由，我们会在之后进行修改 + +""" + +FORMAT_EXAMPLE = """ + +## Code Review All + +### a.py +- 它少实现了xxx需求... +- 字段yyy没有给出... +- ... + +### b.py +... + +### c.py +... + +## Call flow +```mermaid +flowchart TB + c1-->a2 + subgraph one + a1-->a2 + end + subgraph two + b1-->b2 + end + subgraph three + c1-->c2 + end +``` + +## Summary +- a.py:... +- b.py:... +- c.py:... +- ... + +## TODOs +{ + "a.py": "implement requirement xxx...", +} + +""" + + +class SummarizeCode(Action): + def __init__(self, name="SummarizeCode", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + async def summarize_code(self, prompt): + code_rsp = await self._aask(prompt) + return code_rsp + + async def run(self, context): + format_example = FORMAT_EXAMPLE + prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) + logger.info("Summarize code..") + rsp = await self.summarize_code(prompt) + return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..2631ec138 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,32 +5,36 @@ @Author : alexanderwu @File : write_code.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. - ----- # Context {context} ----- + +## Code: {filename} Write code with triple quoto, based on the following list and context. +1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. +5. Think before writing: What should be implemented and provided in this document? +6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +7. Do not use public member functions that do not exist in your design. +8. Before using a variable, make sure you reference it first +9. Write out EVERY DETAIL, DON'T LEAVE TODO. + ## Format example ----- ## Code: {filename} @@ -57,8 +61,8 @@ class WriteCode(Action): design = [i for i in context if i.cause_by == WriteDesign][0] - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name + ws_name = CodeParser.parse_str(block="project_name", text=design.content) + ws_path = CONFIG.workspace_path / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): ws_path = ws_path / ws_name code_path = ws_path / filename diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 4ff4d6cf6..aebe3f4fa 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -6,58 +6,84 @@ @File : write_code_review.py """ +from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed +from metagpt.config import CONFIG PROMPT_TEMPLATE = """ NOTICE Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` - -## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. ------ # Context {context} -## Code: {filename} +## Code to be Reviewed: {filename} ``` {code} ``` + ----- +## Code Review: Based on the "Code to be Reviewed", provide key, clear, concise, and specific code modification suggestions, up to 5. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Is the code logic completely correct? If there are errors, please indicate how to correct them. +3. Does the existing code follow the "Data structures and interfaces"? +4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step. +5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported +6. Is the code implemented concisely enough? Are methods from other files being reused correctly? + +## Code Review Result: If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM. +LGTM/LBTM + +## Rewrite Code: if it still has some bugs, rewrite {filename} based on "Code Review" with triple quotes, try to get LGTM. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. RETURN ALL CODE, NEVER OMIT ANYTHING. 以任何方式省略代码都是不允许的。 +``` +``` + ## Format example ------ {format_example} ------ """ FORMAT_EXAMPLE = """ - -## Code Review -1. The code ... +----- +# EXAMPLE 1 +## Code Review: {filename} +1. No, we should add the logic of ... 2. ... 3. ... 4. ... 5. ... +6. ... + +## Code Review Result: {filename} +LBTM ## Rewrite Code: {filename} ```python ## {filename} ... ``` +----- +# EXAMPLE 2 +## Code Review: {filename} +1. Yes. +2. Yes. +3. Yes. +4. Yes. +5. Yes. +6. Yes. + +## Code Review Result: {filename} +LGTM + +## Rewrite Code: {filename} +pass +----- """ @@ -66,17 +92,27 @@ class WriteCodeReview(Action): super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def write_code(self, prompt): + async def write_code_review_and_rewrite(self, prompt): code_rsp = await self._aask(prompt) + result = CodeParser.parse_block("Code Review Result", code_rsp) + if "LGTM" in result: + return result, None code = CodeParser.parse_code(block="", text=code_rsp) - return code + return result, code async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) - logger.info(f'Code review {filename}..') - code = await self.write_code(prompt) + iterative_code = code + k = CONFIG.code_review_k_times + for i in range(k): + format_example = FORMAT_EXAMPLE.format(filename=filename) + prompt = PROMPT_TEMPLATE.format(context=context, code=iterative_code, filename=filename, format_example=format_example) + logger.info(f'Code review and rewrite {filename}: {i+1}/{k} | {len(iterative_code)=}, {len(code)=}') + result, rewrited_code = await self.write_code_review_and_rewrite(prompt) + if "LBTM" in result: + iterative_code = rewrited_code + elif "LGTM" in result: + return iterative_code # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) - return code - \ No newline at end of file + # 如果rewrited_code是None（原code perfect），那么直接返回code + return iterative_code diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..4780762ca 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -17,53 +17,50 @@ templates = { "json": { "PROMPT_TEMPLATE": """ # Context -## Original Requirements -{requirements} - -## Search Information -{search_information} - -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` +{{ + "Original Requirements": "{requirements}", + "Search Information": "" +}} ## Format example {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. +ATTENTION: Output carefully referenced "Format example" in format. -## Original Requirements: Provide as Plain text, place the polished complete original requirements here +## YOU NEED TO FULFILL THE BELOW JSON DOC -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +{{ + "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. + "Original Requirements": "", # str, place the polished complete original requirements here + "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "Search Information": "", + "Requirements": "", + "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. + "User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories + "Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses + # Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + "Competitive Quadrant Chart": "quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + Campaign A: [0.3, 0.6] + Campaign B: [0.45, 0.23] + Campaign C: [0.57, 0.69] + Campaign D: [0.78, 0.34] + Campaign E: [0.40, 0.34] + Campaign F: [0.35, 0.78]", + "Requirement Analysis": "", # Provide as Plain text. + "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards + "UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. + "Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it. +}} output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -71,6 +68,7 @@ and only output the json inside this tag, nothing else "FORMAT_EXAMPLE": """ [CONTENT] { + "Language": "", "Original Requirements": "", "Search Information": "", "Requirements": "", @@ -131,30 +129,33 @@ quadrantChart {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design +Language: Please use the same language as the user requirement to answer, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese. +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. +## Language: Provide as Plain text, use the same language as the user requirement. + ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories ## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ --- ## Original Requirements -The boss ... +The user ... ## Product Goals ```python @@ -206,6 +207,7 @@ There are no unclear points. } OUTPUT_MAPPING = { + "Language": (str, ...), "Original Requirements": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), @@ -231,11 +233,14 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) + # logger.info(format) prompt_template, format_example = get_template(templates, format) + # logger.info(prompt_template) + # logger.info(format_example) prompt = prompt_template.format( requirements=requirements, search_information=info, format_example=format_example ) - logger.debug(prompt) + # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..9988fda16 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -3,7 +3,7 @@ """ @Time : 2023/5/11 22:12 @Author : alexanderwu -@File : environment.py +@File : write_test.py """ from metagpt.actions.action import Action from metagpt.logs import logger @@ -15,7 +15,7 @@ NOTICE 2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. 3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. 4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. 6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? 7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. diff --git a/metagpt/config.py b/metagpt/config.py index 3f9e742bd..d30a337e3 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -8,7 +8,9 @@ import os import openai import yaml -from metagpt.const import PROJECT_ROOT +from pathlib import Path + +from metagpt.const import METAGPT_ROOT, DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -35,13 +37,14 @@ class Config(metaclass=Singleton): """ _instance = None - key_yaml_file = PROJECT_ROOT / "config/key.yaml" - default_yaml_file = PROJECT_ROOT / "config/config.yaml" + home_yaml_file = Path.home() / ".metagpt/config.yaml" + key_yaml_file = METAGPT_ROOT / "config/key.yaml" + default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) - logger.info("Config loading done.") + # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") @@ -51,10 +54,7 @@ class Config(metaclass=Singleton): (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy - openai.api_base = self.openai_api_base + self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) @@ -84,6 +84,7 @@ class Config(metaclass=Singleton): logger.warning("LONG_TERM_MEMORY is True") self.max_budget = self._get("MAX_BUDGET", 10.0) self.total_cost = 0.0 + self.code_review_k_times = 2 self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") self.mmdc = self._get("MMDC", "mmdc") @@ -94,12 +95,18 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) + self._ensure_workspace_exists() + + def _ensure_workspace_exists(self): + self.workspace_path.mkdir(parents=True, exist_ok=True) + logger.info(f"WORKSPACE_PATH set to {self.workspace_path}") def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" configs.update(os.environ) - for _yaml_file in [yaml_file, self.key_yaml_file]: + for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]: if not _yaml_file.exists(): continue diff --git a/metagpt/const.py b/metagpt/const.py index 407ce803a..14e692487 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -5,44 +5,54 @@ @Author : alexanderwu @File : const.py """ +import os from pathlib import Path from loguru import logger - -def get_project_root(): - """Search upwards to find the project root directory.""" - current_path = Path.cwd() - while True: - if ( - (current_path / ".git").exists() - or (current_path / ".project_root").exists() - or (current_path / ".gitignore").exists() - ): - # use metagpt with git clone will land here - logger.info(f"PROJECT_ROOT set to {str(current_path)}") - return current_path - parent_path = current_path.parent - if parent_path == current_path: - # use metagpt with pip install will land here - cwd = Path.cwd() - logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") - return cwd - current_path = parent_path +import metagpt -PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / "data" -WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -UT_PATH = PROJECT_ROOT / "data/ut" -SWAGGER_PATH = UT_PATH / "files/api/" -UT_PY_PATH = UT_PATH / "files/ut/" -API_QUESTIONS_PATH = UT_PATH / "files/question/" -YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / "tmp" +def get_metagpt_package_root(): + """Get the root directory of the installed package.""" + package_root = Path(metagpt.__file__).parent.parent + logger.info(f"Package root set to {str(package_root)}") + return package_root + + +def get_metagpt_root(): + """Get the project root directory.""" + # Check if a project root is specified in the environment variable + project_root_env = os.getenv('METAGPT_PROJECT_ROOT') + if project_root_env: + project_root = Path(project_root_env) + logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}") + else: + # Fallback to package root if no environment variable is set + project_root = get_metagpt_package_root() + return project_root + + +# METAGPT PROJECT ROOT AND VARS + +METAGPT_ROOT = get_metagpt_root() +DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace" + +DATA_PATH = METAGPT_ROOT / "data" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table" +UT_PATH = DATA_PATH / "ut" +SWAGGER_PATH = UT_PATH / "files/api/" +UT_PY_PATH = UT_PATH / "files/ut/" +API_QUESTIONS_PATH = UT_PATH / "files/question/" -SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" +TMP = METAGPT_ROOT / "tmp" + +SOURCE_ROOT = METAGPT_ROOT / "metagpt" +PROMPT_PATH = SOURCE_ROOT / "prompts" +SKILL_DIRECTORY = SOURCE_ROOT / "skills" + + +# REAL CONSTS MEM_TTL = 24 * 30 * 3600 +YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/document.py b/metagpt/document.py new file mode 100644 index 000000000..cf0821421 --- /dev/null +++ b/metagpt/document.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/6/8 14:03 +@Author : alexanderwu +@File : document.py +""" +from enum import Enum +from typing import Union, Optional +from pathlib import Path +from pydantic import BaseModel, Field +import pandas as pd +from langchain.document_loaders import ( + TextLoader, + UnstructuredPDFLoader, + UnstructuredWordDocumentLoader, +) +from langchain.text_splitter import CharacterTextSplitter +from tqdm import tqdm + +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.repo_parser import RepoParser + + +def validate_cols(content_col: str, df: pd.DataFrame): + if content_col not in df.columns: + raise ValueError("Content column not found in DataFrame.") + + +def read_data(data_path: Path): + suffix = data_path.suffix + if '.xlsx' == suffix: + data = pd.read_excel(data_path) + elif '.csv' == suffix: + data = pd.read_csv(data_path) + elif '.json' == suffix: + data = pd.read_json(data_path) + elif suffix in ('.docx', '.doc'): + data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() + elif '.txt' == suffix: + data = TextLoader(str(data_path)).load() + text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) + texts = text_splitter.split_documents(data) + data = texts + elif '.pdf' == suffix: + data = UnstructuredPDFLoader(str(data_path), mode="elements").load() + else: + raise NotImplementedError("File format not supported.") + return data + + +class DocumentStatus(Enum): + """Indicates document status, a mechanism similar to RFC/PEP""" + DRAFT = "draft" + UNDERREVIEW = "underreview" + APPROVED = "approved" + DONE = "done" + + +class Document(BaseModel): + """ + Document: Handles operations related to document files. + """ + path: Path = Field(default=None) + name: str = Field(default="") + content: str = Field(default="") + + # metadata? in content perhaps. + author: str = Field(default="") + status: DocumentStatus = Field(default=DocumentStatus.DRAFT) + reviews: list = Field(default_factory=list) + + @classmethod + def from_path(cls, path: Path): + """ + Create a Document instance from a file path. + """ + if not path.exists(): + raise FileNotFoundError(f"File {path} not found.") + content = path.read_text() + return cls(content=content, path=path) + + @classmethod + def from_text(cls, text: str, path: Optional[Path] = None): + """ + Create a Document from a text string. + """ + return cls(content=text, path=path) + + def to_path(self, path: Optional[Path] = None): + """ + Save content to the specified file path. + """ + if path is not None: + self.path = path + + if self.path is None: + raise ValueError("File path is not set.") + + self.path.parent.mkdir(parents=True, exist_ok=True) + self.path.write_text(self.content, encoding="utf-8") + + def persist(self): + """ + Persist document to disk. + """ + return self.to_path() + + +class IndexableDocument(Document): + """ + Advanced document handling: For vector databases or search engines. + """ + data: Union[pd.DataFrame, list] + content_col: Optional[str] = Field(default='') + meta_col: Optional[str] = Field(default='') + + class Config: + arbitrary_types_allowed = True + + @classmethod + def from_path(cls, data_path: Path, content_col='content', meta_col='metadata'): + if not data_path.exists(): + raise FileNotFoundError(f"File {data_path} not found.") + data = read_data(data_path) + content = data_path.read_text() + if isinstance(data, pd.DataFrame): + validate_cols(content_col, data) + return cls(data=data, content=content, content_col=content_col, meta_col=meta_col) + + def _get_docs_and_metadatas_by_df(self) -> (list, list): + df = self.data + docs = [] + metadatas = [] + for i in tqdm(range(len(df))): + docs.append(df[self.content_col].iloc[i]) + if self.meta_col: + metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) + else: + metadatas.append({}) + return docs, metadatas + + def _get_docs_and_metadatas_by_langchain(self) -> (list, list): + data = self.data + docs = [i.page_content for i in data] + metadatas = [i.metadata for i in data] + return docs, metadatas + + def get_docs_and_metadatas(self) -> (list, list): + if isinstance(self.data, pd.DataFrame): + return self._get_docs_and_metadatas_by_df() + elif isinstance(self.data, list): + return self._get_docs_and_metadatas_by_langchain() + else: + raise NotImplementedError("Data type not supported for metadata extraction.") + + +class RepoMetadata(BaseModel): + + name: str = Field(default="") + n_docs: int = Field(default=0) + n_chars: int = Field(default=0) + symbols: list = Field(default_factory=list) + + +class Repo(BaseModel): + + # Name of this repo. + name: str = Field(default="") + # metadata: RepoMetadata = Field(default=RepoMetadata) + docs: dict[Path, Document] = Field(default_factory=dict) + codes: dict[Path, Document] = Field(default_factory=dict) + assets: dict[Path, Document] = Field(default_factory=dict) + path: Path = Field(default=None) + + def _path(self, filename): + return self.path / filename + + @classmethod + def from_path(cls, path: Path): + """Load documents, code, and assets from a repository path.""" + path.mkdir(parents=True, exist_ok=True) + repo = Repo(path=path, name=path.name) + for file_path in path.rglob('*'): + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if file_path.is_file() and file_path.suffix in [".json", ".txt", ".md", ".py", ".js", ".css", ".html"]: + repo._set(file_path.read_text(), file_path) + return repo + + def to_path(self): + """Persist all documents, code, and assets to the given repository path.""" + for doc in self.docs.values(): + doc.to_path() + for code in self.codes.values(): + code.to_path() + for asset in self.assets.values(): + asset.to_path() + + def _set(self, content: str, path: Path): + """Add a document to the appropriate category based on its file extension.""" + suffix = path.suffix + doc = Document(content=content, path=path, name=str(path.relative_to(self.path))) + + # FIXME: These judgments are difficult to support multiple programming languages and need to be more general + if suffix.lower() == '.md': + self.docs[path] = doc + elif suffix.lower() in ['.py', '.js', '.css', '.html']: + self.codes[path] = doc + else: + self.assets[path] = doc + return doc + + def set(self, content: str, filename: str): + """Set a document and persist it to disk.""" + path = self._path(filename) + doc = self._set(content, path) + doc.to_path() + + def get(self, filename: str) -> Optional[Document]: + """Get a document by its filename.""" + path = self._path(filename) + return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + + def get_text_documents(self) -> list[Document]: + return list(self.docs.values()) + list(self.codes.values()) + + def eda(self) -> RepoMetadata: + n_docs = sum(len(i) for i in [self.docs, self.codes, self.assets]) + n_chars = sum(sum(len(j.content) for j in i.values()) for i in [self.docs, self.codes, self.assets]) + symbols = RepoParser(base_directory=self.path).generate_symbols() + return RepoMetadata(name=self.name, n_docs=n_docs, n_chars=n_chars, symbols=symbols) + + +def set_existing_repo(path=CONFIG.workspace_path / "t1"): + repo1 = Repo.from_path(path) + repo1.set("wtf content", "doc/wtf_file.md") + repo1.set("wtf code", "code/wtf_file.py") + logger.info(repo1) # check doc + + +def load_existing_repo(path=CONFIG.workspace_path / "web_tetris"): + repo = Repo.from_path(path) + logger.info(repo) + logger.info(repo.eda()) + + +def main(): + load_existing_repo() + + +if __name__ == '__main__': + main() diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 5d7015e8b..84b47a98c 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -28,20 +28,20 @@ class BaseStore(ABC): class LocalStore(BaseStore, ABC): - def __init__(self, raw_data: Path, cache_dir: Path = None): - if not raw_data: + def __init__(self, raw_data_path: Path, cache_dir: Path = None): + if not raw_data_path: raise FileNotFoundError self.config = Config() - self.raw_data = raw_data + self.raw_data_path = raw_data_path if not cache_dir: - cache_dir = raw_data.parent + cache_dir = raw_data_path.parent self.cache_dir = cache_dir self.store = self._load() if not self.store: self.store = self.write() def _get_index_and_store_fname(self): - fname = self.raw_data.name.split('.')[0] + fname = self.raw_data_path.name.split('.')[0] index_file = self.cache_dir / f"{fname}.index" store_file = self.cache_dir / f"{fname}.pkl" return index_file, store_file diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py deleted file mode 100644 index e4b9473c7..000000000 --- a/metagpt/document_store/document.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/6/8 14:03 -@Author : alexanderwu -@File : document.py -""" -from pathlib import Path - -import pandas as pd -from langchain.document_loaders import ( - TextLoader, - UnstructuredPDFLoader, - UnstructuredWordDocumentLoader, -) -from langchain.text_splitter import CharacterTextSplitter -from tqdm import tqdm - - -def validate_cols(content_col: str, df: pd.DataFrame): - if content_col not in df.columns: - raise ValueError - - -def read_data(data_path: Path): - suffix = data_path.suffix - if '.xlsx' == suffix: - data = pd.read_excel(data_path) - elif '.csv' == suffix: - data = pd.read_csv(data_path) - elif '.json' == suffix: - data = pd.read_json(data_path) - elif suffix in ('.docx', '.doc'): - data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() - elif '.txt' == suffix: - data = TextLoader(str(data_path)).load() - text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) - texts = text_splitter.split_documents(data) - data = texts - elif '.pdf' == suffix: - data = UnstructuredPDFLoader(str(data_path), mode="elements").load() - else: - raise NotImplementedError - return data - - -class Document: - - def __init__(self, data_path, content_col='content', meta_col='metadata'): - self.data = read_data(data_path) - if isinstance(self.data, pd.DataFrame): - validate_cols(content_col, self.data) - self.content_col = content_col - self.meta_col = meta_col - - def _get_docs_and_metadatas_by_df(self) -> (list, list): - df = self.data - docs = [] - metadatas = [] - for i in tqdm(range(len(df))): - docs.append(df[self.content_col].iloc[i]) - if self.meta_col: - metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) - else: - metadatas.append({}) - - return docs, metadatas - - def _get_docs_and_metadatas_by_langchain(self) -> (list, list): - data = self.data - docs = [i.page_content for i in data] - metadatas = [i.metadata for i in data] - return docs, metadatas - - def get_docs_and_metadatas(self) -> (list, list): - if isinstance(self.data, pd.DataFrame): - return self._get_docs_and_metadatas_by_df() - elif isinstance(self.data, list): - return self._get_docs_and_metadatas_by_langchain() - else: - raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index dd450010d..885ad3e15 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -15,15 +15,15 @@ from langchain.vectorstores import FAISS from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore -from metagpt.document_store.document import Document +from metagpt.document import IndexableDocument from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data_path: Path, cache_dir=None, meta_col='source', content_col='output'): self.meta_col = meta_col self.content_col = content_col - super().__init__(raw_data, cache_dir) + super().__init__(raw_data_path, cache_dir) def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() @@ -60,9 +60,9 @@ class FaissStore(LocalStore): def write(self): """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user.""" - if not self.raw_data.exists(): + if not self.raw_data_path.exists(): raise FileNotFoundError - doc = Document(self.raw_data, self.content_col, self.meta_col) + doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col) docs, metadatas = doc.get_docs_and_metadatas() self.store = self._write(docs, metadatas) diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..44c9b1c67 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -7,23 +7,28 @@ """ import asyncio from typing import Iterable +from pathlib import Path from pydantic import BaseModel, Field +# from metagpt.document import Document +from metagpt.logs import logger +from metagpt.document import Repo from metagpt.memory import Memory from metagpt.roles import Role from metagpt.schema import Message class Environment(BaseModel): - """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + """ + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles """ roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) history: str = Field(default='') + repo: Repo = Field(default_factory=Repo) + kv: dict = Field(default_factory=dict) class Config: arbitrary_types_allowed = True @@ -50,6 +55,33 @@ class Environment(BaseModel): self.memory.add(message) self.history += f"\n{message}" + def set_doc(self, content: str, filename: str): + """向当前环境发布文档（包括代码）""" + return self.repo.set(content, filename) + + def get_doc(self, filename: str): + return self.repo.get(filename) + + def set(self, k: str, v: str): + self.kv[k] = v + + def get(self, k: str): + return self.kv.get(k, None) + + def load_existing_repo(self, path: Path, inc: bool): + self.repo = Repo.from_path(path) + logger.info(self.repo.eda()) + + # Incremental mode: publish all docs to messages. Then roles can read the docs. + if inc: + docs = self.repo.get_text_documents() + for doc in docs: + msg = Message(content=doc.content) + self.publish_message(msg) + logger.info(f"Message from existing doc {doc.path}: {msg}") + logger.info(f"Load {len(docs)} docs from existing repo.") + raise NotImplementedError + async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once diff --git a/metagpt/logs.py b/metagpt/logs.py index b2052e9b8..afebbfed9 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -10,15 +10,15 @@ import sys from loguru import logger as _logger -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT + def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """调整日志级别到level之上 - Adjust the log level to above level - """ + """Adjust the log level to above level""" _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + _logger.add(METAGPT_ROOT / 'logs/log.txt', level=logfile_level) return _logger + logger = define_log_level() diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..7cbbe651e 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -14,7 +14,7 @@ class Manager: def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { - "BOSS": "Product Manager", + "User": "Product Manager", "Product Manager": "Architect", "Architect": "Engineer", "Engineer": "QA Engineer", diff --git a/metagpt/memory/__init__.py b/metagpt/memory/__init__.py index 710930626..bd6e72163 100644 --- a/metagpt/memory/__init__.py +++ b/metagpt/memory/__init__.py @@ -7,10 +7,10 @@ """ from metagpt.memory.memory import Memory -from metagpt.memory.longterm_memory import LongTermMemory +# from metagpt.memory.longterm_memory import LongTermMemory __all__ = [ "Memory", - "LongTermMemory", + # "LongTermMemory", ] diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index f8abea5f3..b21f80b7d 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -28,7 +28,7 @@ class LongTermMemory(Memory): logger.warning(f"It may the first time to run Agent {role_id}, the long-term memory is empty") else: logger.warning( - f"Agent {role_id} has existed memory storage with {len(messages)} messages " f"and has recovered them." + f"Agent {role_id} has existing memory storage with {len(messages)} messages " f"and has recovered them." ) self.msg_from_recover = True self.add_batch(messages) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 34e5693f8..8ac0c4b21 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -157,6 +157,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if config.openai_api_type: openai.api_type = config.openai_api_type openai.api_version = config.openai_api_version + if config.openai_proxy: + openai.proxy = config.openai_proxy self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: diff --git a/metagpt/repo_parser.py b/metagpt/repo_parser.py new file mode 100644 index 000000000..0020d47aa --- /dev/null +++ b/metagpt/repo_parser.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/17 17:58 +@Author : alexanderwu +@File : repo_parser.py +""" +import json +from pathlib import Path + +import ast +import pandas as pd +from pydantic import BaseModel, Field +from pprint import pformat + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class RepoParser(BaseModel): + base_directory: Path = Field(default=None) + + def parse_file(self, file_path): + """Parse a Python file in the repository.""" + try: + return ast.parse(file_path.read_text()).body + except: + return [] + + def extract_class_and_function_info(self, tree, file_path): + """Extract class, function, and global variable information from the AST.""" + file_info = { + "file": str(file_path.relative_to(self.base_directory)), + "classes": [], + "functions": [], + "globals": [] + } + + for node in tree: + if isinstance(node, ast.ClassDef): + class_methods = [m.name for m in node.body if is_func(m)] + file_info["classes"].append({"name": node.name, "methods": class_methods}) + elif is_func(node): + file_info["functions"].append(node.name) + elif isinstance(node, (ast.Assign, ast.AnnAssign)): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + file_info["globals"].append(target.id) + return file_info + + def generate_symbols(self): + files_classes = [] + directory = self.base_directory + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + return files_classes + + def generate_json_structure(self, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = self.generate_symbols() + output_path.write_text(json.dumps(files_classes, indent=4)) + + def generate_dataframe_structure(self, output_path): + """Generate a DataFrame documenting the repository structure and save as CSV.""" + files_classes = self.generate_symbols() + df = pd.DataFrame(files_classes) + df.to_csv(output_path, index=False) + + def generate_structure(self, output_path=None, mode='json'): + """Generate the structure of the repository as a specified format.""" + output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" + output_path = Path(output_path) if output_path else output_file + + if mode == 'json': + self.generate_json_structure(output_path) + elif mode == 'csv': + self.generate_dataframe_structure(output_path) + + +def is_func(node): + return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + + +def main(): + repo_parser = RepoParser(base_directory=CONFIG.workspace_path / "web_2048") + symbols = repo_parser.generate_symbols() + logger.info(pformat(symbols)) + + +if __name__ == '__main__': + main() diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 1f6685b38..e3f36b50d 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,7 +11,8 @@ from collections import OrderedDict from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -73,35 +74,35 @@ class Engineer(Role): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: - self._init_actions([WriteCode, WriteCodeReview]) + # if self.use_code_review: + # self._init_actions([WriteCode, WriteCodeReview]) self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: + def parse_tasks(cls, task_msg: Message) -> list[str]: if task_msg.instruct_content: return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod - def parse_code(self, code_text: str) -> str: + def parse_code(cls, code_text: str) -> str: return CodeParser.parse_code(block="", text=code_text) @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + return system_design_msg.instruct_content.dict().get("project_name").strip().strip("'").strip('"') + return CodeParser.parse_str(block="project_name", text=system_design_msg.content) def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def recreate_workspace(self): workspace = self.get_workspace() @@ -167,7 +168,7 @@ class Engineer(Role): ) return msg - async def _act_sp_precision(self) -> Message: + async def _act_sp_with_cr(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: """ @@ -181,17 +182,16 @@ class Engineer(Role): msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) for m in msg: context.append(m.content) - context_str = "\n".join(context) + context_str = "\n----------\n".join(context) # Write code code = await WriteCode().run(context=context_str, filename=todo) # Code review if self.use_code_review: - try: - rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass + # try: + rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) + code = rewrite_code + # except Exception as e: + # logger.error("code review failed!", e) file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -199,6 +199,13 @@ class Engineer(Role): code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n----------\n".join(context) + summary = await SummarizeCode().run(context=context_str) + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" @@ -209,5 +216,5 @@ class Engineer(Role): """Determines the mode of action based on whether code review is used.""" logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: - return await self._act_sp_precision() + return await self._act_sp_with_cr() return await self._act_sp() diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..f6172b607 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : product_manager.py """ -from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions import UserRequirement, WritePRD from metagpt.roles import Role @@ -38,4 +38,4 @@ class ProductManager(Role): """ super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..313fe4aba 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -16,7 +16,8 @@ from metagpt.actions import ( WriteDesign, WriteTest, ) -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -44,19 +45,19 @@ class QaEngineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name") - return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) + return system_design_msg.instruct_content.dict().get("project_name") + return CodeParser.parse_str(block="project_name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace_path / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b96c361c0..5c5e7b76d 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -17,7 +17,8 @@ from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import Memory +# from metagpt.memory import LongTermMemory from metagpt.schema import Message PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -49,6 +50,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi {name}: {result} """ + class RoleReactMode(str, Enum): REACT = "react" BY_ORDER = "by_order" @@ -58,6 +60,7 @@ class RoleReactMode(str, Enum): def values(cls): return [item.value for item in cls] + class RoleSetting(BaseModel): """Role Settings""" name: str @@ -78,7 +81,7 @@ class RoleContext(BaseModel): """Role Runtime Context""" env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) - long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) + # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) @@ -130,6 +133,7 @@ class Role: logger.warning(f"is_human attribute does not take effect," f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances") i = action + i.set_env(self._rc.env) i.set_prefix(self._get_prefix(), self.profile) self._actions.append(i) self._states.append(f"{idx}. {action}") @@ -171,6 +175,18 @@ class Role: """Set the environment in which the role works. The role can talk to the environment and can also receive messages by observing.""" self._rc.env = env + def set_doc(self, content: str, filename: str): + return self._rc.env.set_doc(content, filename) + + def get_doc(self, filename: str): + return self._rc.env.get_doc(filename) + + def set(self, k, v): + return self._rc.env.set(k, v) + + def get(self, k): + return self._rc.env.get(k) + @property def profile(self): """Get the role description (position)""" diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index b27841d74..4069f4836 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -9,7 +9,7 @@ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner from semantic_kernel.planning.basic_planner import BasicPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions.execute_task import ExecuteTask from metagpt.logs import logger from metagpt.roles import Role @@ -39,7 +39,7 @@ class SkAgent(Role): """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([ExecuteTask()]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) self.kernel = make_sk_kernel() # how funny the interface is inconsistent diff --git a/metagpt/software_company.py b/metagpt/software_company.py deleted file mode 100644 index d44a0068a..000000000 --- a/metagpt/software_company.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/12 00:30 -@Author : alexanderwu -@File : software_company.py -""" -from metagpt.team import Team as SoftwareCompany - -import warnings -warnings.warn("metagpt.software_company is deprecated and will be removed in the future" - "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", - DeprecationWarning, 2) diff --git a/metagpt/startup.py b/metagpt/startup.py new file mode 100644 index 000000000..38f457fc2 --- /dev/null +++ b/metagpt/startup.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from pathlib import Path +import asyncio +import typer + +app = typer.Typer() + + +@app.command() +def startup( + idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), + investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(True, help="Whether to use code review."), + run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(True, help="Enable or disable code implementation."), + project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."), + inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."), +): + """Run a startup. Be a boss.""" + from metagpt.roles import ProductManager, Architect, ProjectManager, Engineer, QaEngineer + from metagpt.team import Team + + company = Team() + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) + + if implement or code_review: + company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + + if run_tests: + company.hire([QaEngineer()]) + + company.invest(investment) + company.run_project(idea, project_name=project_name, inc=inc) + asyncio.run(company.run(n_round=n_round)) + + +if __name__ == "__main__": + startup(idea="Make a 2048 game.") diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..a22a09fe4 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -7,7 +7,7 @@ """ from pydantic import BaseModel, Field -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger @@ -21,7 +21,7 @@ class Team(BaseModel): Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to perform any multi-agent activity, such as collaboratively writing executable code. """ - environment: Environment = Field(default_factory=Environment) + env: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -30,7 +30,7 @@ class Team(BaseModel): def hire(self, roles: list[Role]): """Hire roles to cooperate""" - self.environment.add_roles(roles) + self.env.add_roles(roles) def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" @@ -42,13 +42,19 @@ class Team(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea, send_to: str = ""): - """Start a project from publishing boss requirement.""" + def run_project(self, idea, send_to: str = "", project_name: str = "", inc: bool = False): + """Start a project from publishing user requirement.""" self.idea = idea - self.environment.publish_message(Message(role="Human", content=idea, cause_by=BossRequirement, send_to=send_to)) + # If user set project_name, then use it. + if project_name: + path = CONFIG.workspace_path / project_name + self.env.load_existing_repo(path, inc=inc) + + # Human requirement. + self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) def _save(self): - logger.info(self.json()) + logger.info(self.json(ensure_ascii=False)) async def run(self, n_round=3): """Run company until target round or no money""" @@ -57,6 +63,6 @@ class Team(BaseModel): n_round -= 1 logger.debug(f"{n_round=}") self._check_balance() - await self.environment.run() - return self.environment.history + await self.env.run() + return self.env.history \ No newline at end of file diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 1d9cd0b2a..4f010a912 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -13,12 +13,10 @@ from typing import List from aiohttp import ClientSession from PIL import Image, PngImagePlugin -from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +# from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -config = Config() - payload = { "prompt": "", "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", @@ -56,9 +54,8 @@ default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" class SDEngine: def __init__(self): # Initialize the SDEngine with configuration - self.config = Config() - self.sd_url = self.config.get("SD_URL") - self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}" + self.sd_url = CONFIG.get("SD_URL") + self.sd_t2i_url = f"{self.sd_url}{CONFIG.get('SD_T2I_API')}" # Define default payload settings for SD API self.payload = payload logger.info(self.sd_t2i_url) @@ -81,7 +78,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.workspace_path / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) @@ -125,6 +122,7 @@ def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) + if __name__ == "__main__": engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 204c22c67..eb85a3f90 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -10,7 +10,7 @@ import os from pathlib import Path from metagpt.config import CONFIG -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists @@ -69,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, if stdout: logger.info(stdout.decode()) if stderr: - logger.error(stderr.decode()) + logger.warning(stderr.decode()) else: if engine == "playwright": from metagpt.utils.mmdc_playwright import mermaid_to_file @@ -141,6 +141,6 @@ MMC2 = """sequenceDiagram if __name__ == "__main__": loop = asyncio.new_event_loop() - result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) - result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) loop.close() diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 1af96f272..33bcd01a5 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -21,6 +21,7 @@ TOKEN_COSTS = { "gpt-4-32k": {"prompt": 0.06, "completion": 0.12}, "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, + "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ TOKEN_MAX = { "gpt-4-32k": 32768, "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, + "gpt-4-1106-preview": 128000, "text-embedding-ada-002": 8192, "chatglm_turbo": 32768 } @@ -56,16 +58,17 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613", + "gpt-4-1106-preview", }: tokens_per_message = 3 tokens_per_name = 1 elif model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n tokens_per_name = -1 # if there's a name, the role is omitted - elif "gpt-3.5-turbo" in model: + elif "gpt-3.5-turbo" == model: print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.") return count_message_tokens(messages, model="gpt-3.5-turbo-0613") - elif "gpt-4" in model: + elif "gpt-4" == model: print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.") return count_message_tokens(messages, model="gpt-4-0613") else: diff --git a/requirements.txt b/requirements.txt index f0169d7fa..f233e398f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,8 @@ channels==4.0.0 # docx==0.2.4 #faiss==1.5.3 faiss_cpu==1.7.4 -fire==0.4.0 +# fire==0.4.0 +typer # godot==0.1.1 # google_api_python_client==2.93.0 lancedb==0.1.16 diff --git a/setup.py b/setup.py index 239156ae3..e7462767f 100644 --- a/setup.py +++ b/setup.py @@ -31,14 +31,14 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", version="0.3.0", - description="The Multi-Role Meta Programming Framework", + description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", - license="Apache 2.0", - keywords="metagpt multi-role multi-agent programming gpt llm", + license="MIT", + keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, @@ -52,4 +52,9 @@ setup( cmdclass={ "install_mermaid": InstallMermaidCLI, }, + entry_points={ + 'console_scripts': [ + 'metagpt=metagpt.startup:app', + ], + }, ) diff --git a/startup.py b/startup.py deleted file mode 100644 index e9fbf94d3..000000000 --- a/startup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.team import Team - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :return: - """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index a800690e8..d367e253e 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -90,7 +90,7 @@ Python's in-built data structures like lists and dictionaries will be used exten For testing, we can use the PyTest framework. This is a mature full-featured Python testing tool that helps you write better programs. -## Python package name: +## project_name: ```python "adventure_game" ``` @@ -100,7 +100,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Pyt file_list = ["main.py", "room.py", "player.py", "game.py", "object.py", "puzzle.py", "test_game.py"] ``` -## Data structures and interface definitions: +## Data structures and interfaces: ```mermaid classDiagram class Room{ @@ -209,7 +209,7 @@ Shared knowledge for this project includes understanding the basic principles of """ ``` -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. ```python """ The original requirements did not specify whether the game should have a save/load feature, multiplayer support, or any specific graphical user interface. More information on these aspects could help in further refining the product design and requirements. diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 38e4e5221..18675ecc3 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -7,7 +7,7 @@ """ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message @@ -17,7 +17,7 @@ from metagpt.schema import Message async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement)) + prd = await product_manager.handle(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) diff --git a/tests/metagpt/document_store/test_document.py b/tests/metagpt/document_store/test_document.py index 5ae357fb1..13c0921a3 100644 --- a/tests/metagpt/document_store/test_document.py +++ b/tests/metagpt/document_store/test_document.py @@ -7,22 +7,22 @@ """ import pytest -from metagpt.const import DATA_PATH -from metagpt.document_store.document import Document +from metagpt.const import METAGPT_ROOT +from metagpt.document import IndexableDocument CASES = [ - ("st/faq.xlsx", "Question", "Answer", 1), - ("cases/faq.csv", "Question", "Answer", 1), + ("requirements.txt", None, None, 0), + # ("cases/faq.csv", "Question", "Answer", 1), # ("cases/faq.json", "Question", "Answer", 1), - ("docx/faq.docx", None, None, 1), - ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 - ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 + # ("docx/faq.docx", None, None, 1), + # ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 + # ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 ] @pytest.mark.parametrize("relative_path, content_col, meta_col, threshold", CASES) def test_document(relative_path, content_col, meta_col, threshold): - doc = Document(DATA_PATH / relative_path, content_col, meta_col) + doc = IndexableDocument.from_path(METAGPT_ROOT / relative_path, content_col, meta_col) rsp = doc.get_docs_and_metadatas() assert len(rsp[0]) > threshold assert len(rsp[1]) > threshold diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..ac9362937 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -4,7 +4,7 @@ from metagpt.config import CONFIG from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.role import RoleContext from metagpt.memory import LongTermMemory @@ -15,24 +15,24 @@ def test_ltm_search(): assert len(openai_api_key) > 20 role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + rc = RoleContext(watch=[UserRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -48,7 +48,7 @@ def test_ltm_search(): assert len(news) == 0 new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..bd4441641 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -6,7 +6,7 @@ from typing import List from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions import WritePRD from metagpt.actions.action_output import ActionOutput @@ -14,7 +14,7 @@ from metagpt.actions.action_output import ActionOutput def test_idea_message(): idea = 'Write a cli snake game' role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -24,12 +24,12 @@ def test_idea_message(): assert memory_storage.is_initialized is True sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) new_messages = memory_storage.search(sim_message) assert len(new_messages) == 0 # similar, return [] new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -49,7 +49,7 @@ def test_actionout_message(): ic_obj = ActionOutput.create_model_class('prd', out_mapping) role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' + content = 'The user has requested the creation of a command-line interface (CLI) snake game' message = Message(content=content, instruct_content=ic_obj(**out_data), role='user', diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 5ab9a493f..8efe6cfc4 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -9,7 +9,7 @@ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -23,7 +23,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 03a82ec5e..f6d44ba03 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -8,7 +8,7 @@ import pytest from semantic_kernel.core_skills import TextSkill -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -26,7 +26,7 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..c06844389 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -5,10 +5,10 @@ @Author : alexanderwu @File : mock.py """ -from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks +from metagpt.actions import UserRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message -BOSS_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" +USER_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" DETAIL_REQUIREMENT = """需求：开发一个基于LLM（大语言模型）与私有知识库的搜索引擎，希望有几点能力 1. 用户可以在私有知识库进行搜索，再根据大语言模型进行总结，输出的结果包括了总结 @@ -71,7 +71,7 @@ PRD = '''## 原始需求 ``` ''' -SYSTEM_DESIGN = '''## Python package name +SYSTEM_DESIGN = '''## project_name ```python "smart_search_engine" ``` @@ -94,7 +94,7 @@ SYSTEM_DESIGN = '''## Python package name ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Main { @@ -252,7 +252,7 @@ a = 'a' class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) + req = Message(role="User", content=USER_REQUIREMENT, cause_by=UserRequirement) prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) diff --git a/tests/metagpt/roles/test_ui.py b/tests/metagpt/roles/test_ui.py index d58d31bd9..ec507f75d 100644 --- a/tests/metagpt/roles/test_ui.py +++ b/tests/metagpt/roles/test_ui.py @@ -18,5 +18,5 @@ async def test_ui_role(idea: str, investment: float = 3.0, n_round: int = 5): company = Team() company.hire([ProductManager(), UI()]) company.invest(investment) - company.start_project(idea) + company.run_project(idea) await company.run(n_round=n_round) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..102c6ebd6 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,8 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -29,7 +30,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Selected Elements:Provide as Plain text, up to 5 specified elements, clear and simple ## HTML Layout:Provide as Plain text, use standard HTML code ## CSS Styles (styles.css):Provide as Plain text,use standard css code -## Anything UNCLEAR:Provide as Plain text. Make clear here. +## Anything UNCLEAR:Provide as Plain text. Try to clarify it. """ @@ -214,7 +215,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace_path / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..b27bc3da7 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -8,7 +8,7 @@ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger from metagpt.manager import Manager @@ -49,7 +49,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) + env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/test_software_company.py b/tests/metagpt/test_startup.py similarity index 51% rename from tests/metagpt/test_software_company.py rename to tests/metagpt/test_startup.py index 4fc651f52..53a8d8735 100644 --- a/tests/metagpt/test_software_company.py +++ b/tests/metagpt/test_startup.py @@ -3,17 +3,26 @@ """ @Time : 2023/5/15 11:40 @Author : alexanderwu -@File : test_software_company.py +@File : test_startup.py """ import pytest +from typer.testing import CliRunner + +runner = CliRunner() from metagpt.logs import logger from metagpt.team import Team +from metagpt.startup import app @pytest.mark.asyncio async def test_team(): company = Team() - company.start_project("做一个基础搜索引擎，可以支持知识库") + company.run_project("做一个基础搜索引擎，可以支持知识库") history = await company.run(n_round=5) logger.info(history) + + +# def test_startup(): +# args = ["Make a 2048 game"] +# result = runner.invoke(app, args) diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..fea58bc29 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,9 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine + def test_sd_engine_init(): @@ -21,5 +23,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace_path / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..b6c000f9b 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -10,7 +10,7 @@ import os import pytest -from metagpt.const import get_project_root +from metagpt.const import get_metagpt_root class TestGetProjectRoot: @@ -20,11 +20,11 @@ class TestGetProjectRoot: os.chdir(abs_root) def test_get_project_root(self): - project_root = get_project_root() + project_root = get_metagpt_root() assert project_root.name == 'metagpt' def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: self.change_etc_dir() - get_project_root() + get_metagpt_root() assert str(exc_info.value) == "Project root not found." diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index 4e362f9f7..99ab1f79e 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -218,7 +218,7 @@ We need clarification on how the high score should be stored. Should it persist } t_text1 = '''## Original Requirements: -The boss wants to create a web-based version of the game "Fly Bird". +The user wants to create a web-based version of the game "Fly Bird". ## Product Goals: diff --git a/tests/metagpt/utils/test_read_docx.py b/tests/metagpt/utils/test_read_docx.py index a7d0774a8..adf473ae7 100644 --- a/tests/metagpt/utils/test_read_docx.py +++ b/tests/metagpt/utils/test_read_docx.py @@ -6,12 +6,12 @@ @File : test_read_docx.py """ -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.utils.read_document import read_docx class TestReadDocx: def test_read_docx(self): - docx_sample = PROJECT_ROOT / "tests/data/docx_for_test.docx" + docx_sample = METAGPT_ROOT / "tests/data/docx_for_test.docx" docx = read_docx(docx_sample) assert len(docx) == 6