diff --git a/examples/agent_creator.py b/examples/agent_creator.py index 325e7c260..bcb9c0c1d 100644 --- a/examples/agent_creator.py +++ b/examples/agent_creator.py @@ -5,13 +5,14 @@ Author: garylin2099 ''' import re -from metagpt.const import PROJECT_ROOT, WORKSPACE_ROOT +from metagpt.const import METAGPT_ROOT +from metagpt.config import CONFIG from metagpt.actions import Action from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger -with open(PROJECT_ROOT / "examples/build_customized_agent.py", "r") as f: +with open(METAGPT_ROOT / "examples/build_customized_agent.py", "r") as f: # use official example script to guide AgentCreator MULTI_ACTION_AGENT_CODE_EXAMPLE = f.read() @@ -49,7 +50,7 @@ class CreateAgent(Action): pattern = r'```python(.*)```' match = re.search(pattern, rsp, re.DOTALL) code_text = match.group(1) if match else "" - with open(WORKSPACE_ROOT / "agent_created_agent.py", "w") as f: + with open(CONFIG.workspace_path / "agent_created_agent.py", "w") as f: f.write(code_text) return code_text diff --git a/examples/debate.py b/examples/debate.py index a37e60848..0f5d1591b 100644 --- a/examples/debate.py +++ b/examples/debate.py @@ -8,7 +8,7 @@ import platform import fire from metagpt.team import Team -from metagpt.actions import Action, BossRequirement +from metagpt.actions import Action, UserRequirement from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger @@ -49,7 +49,7 @@ class Debator(Role): ): super().__init__(name, profile, **kwargs) self._init_actions([SpeakAloud]) - self._watch([BossRequirement, SpeakAloud]) + self._watch([UserRequirement, SpeakAloud]) self.name = name self.opponent_name = opponent_name diff --git a/examples/sk_agent.py b/examples/sk_agent.py index a7513e838..647ea4380 100644 --- a/examples/sk_agent.py +++ b/examples/sk_agent.py @@ -13,7 +13,7 @@ from semantic_kernel.planning import SequentialPlanner # from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -39,7 +39,7 @@ async def basic_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def sequential_planner_example(): @@ -53,7 +53,7 @@ async def sequential_planner_example(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def basic_planner_web_search_example(): @@ -64,7 +64,7 @@ async def basic_planner_web_search_example(): role.import_skill(SkSearchEngine(), "WebSearchSkill") # role.import_semantic_skill_from_directory(skills_directory, "QASkill") - await role.run(Message(content=task, cause_by=BossRequirement)) + await role.run(Message(content=task, cause_by=UserRequirement)) async def action_planner_example(): @@ -75,7 +75,7 @@ async def action_planner_example(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - await role.run(Message(content=task, cause_by=BossRequirement)) # it will choose mathskill.Add + await role.run(Message(content=task, cause_by=UserRequirement)) # it will choose mathskill.Add if __name__ == "__main__": diff --git a/metagpt/actions/SummarizeCode.py b/metagpt/actions/SummarizeCode.py new file mode 100644 index 000000000..1015d3bfb --- /dev/null +++ b/metagpt/actions/SummarizeCode.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Author : alexanderwu +@File : SummarizeCode.py +""" + +from metagpt.actions.action import Action +from metagpt.logs import logger +from metagpt.schema import Message +from metagpt.utils.common import CodeParser +from tenacity import retry, stop_after_attempt, wait_fixed + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to review the code. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +----- +# Context +{context} +----- + +## Code Review All: 请你对历史所有文件进行阅读，分析每个文件是否都完整实现了用户需求，找到可能的bug，如函数未实现、调用错误、未引用等 + +## Summary: 根据历史文件的实现情况进行总结 + +## Call flow: 根据实现的函数，使用mermaid绘制完整的调用链 + +## TODOs: 这里写出需要修改的文件列表，我们会在之后进行修改 + +""" + +FORMAT_EXAMPLE = """ + +## Code Review All + +### a.py +- 它少实现了xxx需求... +- 字段yyy没有给出... +- ... + +### b.py +... + +### c.py +... + +## Call flow +```mermaid +flowchart TB + c1-->a2 + subgraph one + a1-->a2 + end + subgraph two + b1-->b2 + end + subgraph three + c1-->c2 + end +``` + +## Summary +- a.py:... +- b.py:... +- c.py:... +- ... + +## TODOs +1. ... +2. ... +3. ... + +""" + + +class SummarizeCode(Action): + def __init__(self, name="SummaryCode", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + async def write_code_review_all(self, prompt): + code_rsp = await self._aask(prompt) + return code_rsp + + async def run(self, context): + format_example = FORMAT_EXAMPLE.format() + prompt = PROMPT_TEMPLATE.format(context=context, format_example=format_example) + logger.info(f'Code review all..') + rsp = await self.write_code_review_all(prompt) + return rsp + \ No newline at end of file diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index b004bd58e..79ff94b3e 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -9,7 +9,7 @@ from enum import Enum from metagpt.actions.action import Action from metagpt.actions.action_output import ActionOutput -from metagpt.actions.add_requirement import BossRequirement +from metagpt.actions.add_requirement import UserRequirement from metagpt.actions.debug_error import DebugError from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview @@ -28,7 +28,7 @@ from metagpt.actions.write_test import WriteTest class ActionType(Enum): """All types of Actions, used for indexing.""" - ADD_REQUIREMENT = BossRequirement + ADD_REQUIREMENT = UserRequirement WRITE_PRD = WritePRD WRITE_PRD_REVIEW = WritePRDReview WRITE_DESIGN = WriteDesign diff --git a/metagpt/actions/add_requirement.py b/metagpt/actions/add_requirement.py index 7dc09d062..8e2c56a62 100644 --- a/metagpt/actions/add_requirement.py +++ b/metagpt/actions/add_requirement.py @@ -8,7 +8,7 @@ from metagpt.actions import Action -class BossRequirement(Action): - """Boss Requirement without any implementation details""" +class UserRequirement(Action): + """User Requirement without any implementation details""" async def run(self, *args, **kwargs): raise NotImplementedError diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..f58d49495 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,7 +11,6 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -27,21 +26,20 @@ templates = { ## Format example {format_example} ----- -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system Requirement: Fill in the following missing information based on the context, each section name is a key in json -Max Output: 8192 chars or 2048 tokens. Try to use them up. -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -52,7 +50,7 @@ and only output the json inside this tag, nothing else "Implementation approach": "We will ...", "Python package name": "snake_game", "File list": ["main.py"], - "Data structures and interface definitions": ' + "Data structures and interfaces": ' classDiagram class Game{ +int score @@ -81,20 +79,19 @@ and only output the json inside this tag, nothing else ----- Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately -Max Output: 8192 chars or 2048 tokens. Try to use them up. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores +## Python package name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. ## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ @@ -114,7 +111,7 @@ We will ... ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Game{ @@ -143,7 +140,7 @@ OUTPUT_MAPPING = { "Implementation approach": (str, ...), "Python package name": (str, ...), "File list": (List[str], ...), - "Data structures and interface definitions": (str, ...), + "Data structures and interfaces": (str, ...), "Program call flow": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -177,8 +174,8 @@ class WriteDesign(Action): async def _save_system_design(self, docs_path, resources_path, system_design): data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) + "Data structures and interfaces" + ] # CodeParser.parse_code(block="Data structures and interfaces", text=content) seq_flow = system_design.instruct_content.dict()[ "Program call flow" ] # CodeParser.parse_code(block="Program call flow", text=content) @@ -193,7 +190,7 @@ class WriteDesign(Action): ws_name = system_design.instruct_content.dict()["Python package name"] else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name + workspace = CONFIG.workspace_path / ws_name self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index b395fa64e..467cb4d83 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -9,7 +9,6 @@ from typing import List from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -27,9 +26,9 @@ Role: You are a project manager; the goal is to break down tasks according to PR Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Required Python third-party packages: Provided in requirements.txt format +## Required Python third-party packages: Provide Python list[str] in requirements.txt format -## Required Other language third-party packages: Provided in requirements.txt format +## Required Other language third-party packages: Provide Python list[str] in requirements.txt format ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. @@ -39,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -95,7 +94,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. """, "FORMAT_EXAMPLE": ''' @@ -171,11 +170,11 @@ class WriteTasks(Action): ws_name = context[-1].instruct_content.dict()["Python package name"] else: ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" + file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md" file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" + requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt" requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) async def run(self, context, format=CONFIG.prompt_format): diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..176718dfc 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,7 +7,7 @@ """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser @@ -18,19 +18,22 @@ NOTICE Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language) ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code: {filename} Write code with triple quoto, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. -5. Think before writing: What should be implemented and provided in this document? -6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. -7. Do not use public member functions that do not exist in your design. - ----- # Context {context} ----- + +## Code: {filename} Write code with triple quoto, based on the following list and context. +1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets +3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN. +5. Think before writing: What should be implemented and provided in this document? +6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +7. Do not use public member functions that do not exist in your design. +8. Before using a variable, make sure you reference it first +9. Write out EVERY DETAIL, DON'T LEAVE TODO. + ## Format example ----- ## Code: {filename} @@ -58,7 +61,7 @@ class WriteCode(Action): design = [i for i in context if i.cause_by == WriteDesign][0] ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name + ws_path = CONFIG.workspace_path / ws_name if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): ws_path = ws_path / ws_name code_path = ws_path / filename diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 4ff4d6cf6..c6538bf7b 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -17,16 +17,14 @@ NOTICE Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". -## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` +## Code Review: Based on the following context and code, follow the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5. +1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step. +2. Are there any issues with the code logic? If so, how to solve it? +3. Does the existing code follow the "Data structures and interfaces"? +4. Is there a function in the code that is not fully implemented? If so, how to implement it? +5. Does the code have unnecessary or lack dependencies? If so, how to solve it? -## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE. +## Rewrite Code: rewrite {filename} based on "Code Review" with triple quotes. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. ----- # Context {context} @@ -47,7 +45,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc FORMAT_EXAMPLE = """ ## Code Review -1. The code ... +1. No, we should add the logic of ... 2. ... 3. ... 4. ... diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..584d31998 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -46,24 +46,25 @@ quadrantChart {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, each section name is a key in json ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible +## Competitive Analysis: Provided as Python list[str], up to 8 competitive product analyses ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. + +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -131,30 +132,30 @@ quadrantChart {format_example} ----- Role: You are a professional product manager; the goal is to design a concise, usable, efficient product -Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. ## Original Requirements: Provide as Plain text, place the polished complete original requirements here -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories ## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible ## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. +## Requirement Analysis: Provide as Plain text. -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards ## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. """, "FORMAT_EXAMPLE": """ --- ## Original Requirements -The boss ... +The user ... ## Product Goals ```python diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 35ff36dc2..2f4988c09 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -15,7 +15,7 @@ NOTICE 2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases. 3. Attention1: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script. 4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. +5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity. 6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail? 7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. diff --git a/metagpt/config.py b/metagpt/config.py index 3f9e742bd..1a9cdb4d2 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -8,7 +8,9 @@ import os import openai import yaml -from metagpt.const import PROJECT_ROOT +from pathlib import Path + +from metagpt.const import METAGPT_ROOT, DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -35,13 +37,14 @@ class Config(metaclass=Singleton): """ _instance = None - key_yaml_file = PROJECT_ROOT / "config/key.yaml" - default_yaml_file = PROJECT_ROOT / "config/config.yaml" + home_yaml_file = Path.home() / ".metagpt/config.yaml" + key_yaml_file = METAGPT_ROOT / "config/key.yaml" + default_yaml_file = METAGPT_ROOT / "config/config.yaml" def __init__(self, yaml_file=default_yaml_file): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) - logger.info("Config loading done.") + # logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") @@ -94,12 +97,18 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT)) + self._ensure_workspace_exists() + + def _ensure_workspace_exists(self): + self.workspace_path.mkdir(parents=True, exist_ok=True) + logger.info(f"WORKSPACE_PATH set to {self.workspace_path}") def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" configs.update(os.environ) - for _yaml_file in [yaml_file, self.key_yaml_file]: + for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]: if not _yaml_file.exists(): continue diff --git a/metagpt/const.py b/metagpt/const.py index 407ce803a..14e692487 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -5,44 +5,54 @@ @Author : alexanderwu @File : const.py """ +import os from pathlib import Path from loguru import logger - -def get_project_root(): - """Search upwards to find the project root directory.""" - current_path = Path.cwd() - while True: - if ( - (current_path / ".git").exists() - or (current_path / ".project_root").exists() - or (current_path / ".gitignore").exists() - ): - # use metagpt with git clone will land here - logger.info(f"PROJECT_ROOT set to {str(current_path)}") - return current_path - parent_path = current_path.parent - if parent_path == current_path: - # use metagpt with pip install will land here - cwd = Path.cwd() - logger.info(f"PROJECT_ROOT set to current working directory: {str(cwd)}") - return cwd - current_path = parent_path +import metagpt -PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / "data" -WORKSPACE_ROOT = PROJECT_ROOT / "workspace" -PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" -UT_PATH = PROJECT_ROOT / "data/ut" -SWAGGER_PATH = UT_PATH / "files/api/" -UT_PY_PATH = UT_PATH / "files/ut/" -API_QUESTIONS_PATH = UT_PATH / "files/question/" -YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / "tmp" +def get_metagpt_package_root(): + """Get the root directory of the installed package.""" + package_root = Path(metagpt.__file__).parent.parent + logger.info(f"Package root set to {str(package_root)}") + return package_root + + +def get_metagpt_root(): + """Get the project root directory.""" + # Check if a project root is specified in the environment variable + project_root_env = os.getenv('METAGPT_PROJECT_ROOT') + if project_root_env: + project_root = Path(project_root_env) + logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}") + else: + # Fallback to package root if no environment variable is set + project_root = get_metagpt_package_root() + return project_root + + +# METAGPT PROJECT ROOT AND VARS + +METAGPT_ROOT = get_metagpt_root() +DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace" + +DATA_PATH = METAGPT_ROOT / "data" RESEARCH_PATH = DATA_PATH / "research" TUTORIAL_PATH = DATA_PATH / "tutorial_docx" INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table" +UT_PATH = DATA_PATH / "ut" +SWAGGER_PATH = UT_PATH / "files/api/" +UT_PY_PATH = UT_PATH / "files/ut/" +API_QUESTIONS_PATH = UT_PATH / "files/question/" -SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills" +TMP = METAGPT_ROOT / "tmp" + +SOURCE_ROOT = METAGPT_ROOT / "metagpt" +PROMPT_PATH = SOURCE_ROOT / "prompts" +SKILL_DIRECTORY = SOURCE_ROOT / "skills" + + +# REAL CONSTS MEM_TTL = 24 * 30 * 3600 +YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/document.py b/metagpt/document.py new file mode 100644 index 000000000..044210218 --- /dev/null +++ b/metagpt/document.py @@ -0,0 +1,207 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/6/8 14:03 +@Author : alexanderwu +@File : document.py +""" + +from typing import Union, Optional +from pathlib import Path +from pydantic import BaseModel, Field +import pandas as pd +from langchain.document_loaders import ( + TextLoader, + UnstructuredPDFLoader, + UnstructuredWordDocumentLoader, +) +from langchain.text_splitter import CharacterTextSplitter +from tqdm import tqdm + +from metagpt.logs import logger + + +def validate_cols(content_col: str, df: pd.DataFrame): + if content_col not in df.columns: + raise ValueError("Content column not found in DataFrame.") + + +def read_data(data_path: Path): + suffix = data_path.suffix + if '.xlsx' == suffix: + data = pd.read_excel(data_path) + elif '.csv' == suffix: + data = pd.read_csv(data_path) + elif '.json' == suffix: + data = pd.read_json(data_path) + elif suffix in ('.docx', '.doc'): + data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() + elif '.txt' == suffix: + data = TextLoader(str(data_path)).load() + text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) + texts = text_splitter.split_documents(data) + data = texts + elif '.pdf' == suffix: + data = UnstructuredPDFLoader(str(data_path), mode="elements").load() + else: + raise NotImplementedError("File format not supported.") + return data + + +class Document(BaseModel): + """ + Document: Handles operations related to document files. + """ + content: str = Field(default='') + file_path: Path = Field(default=None) + + @classmethod + def from_path(cls, file_path: Path): + """ + Create a Document instance from a file path. + """ + if not file_path.exists(): + raise FileNotFoundError(f"File {file_path} not found.") + content = file_path.read_text() + return cls(content=content, file_path=file_path) + + @classmethod + def from_text(cls, text: str, file_path: Optional[Path] = None): + """ + Create a Document from a text string. + """ + return cls(content=text, file_path=file_path) + + def to_path(self, file_path: Optional[Path] = None): + """ + Save content to the specified file path. + """ + if file_path is not None: + self.file_path = file_path + + if self.file_path is None: + raise ValueError("File path is not set.") + + self.file_path.parent.mkdir(parents=True, exist_ok=True) + self.file_path.write_text(self.content) + + def persist(self): + """ + Persist document to disk. + """ + return self.to_path() + + +class IndexableDocument(Document): + """ + Advanced document handling: For vector databases or search engines. + """ + data: Union[pd.DataFrame, list] + content_col: Optional[str] = Field(default='') + meta_col: Optional[str] = Field(default='') + + class Config: + arbitrary_types_allowed = True + + @classmethod + def from_path(cls, data_path: Path, content_col='content', meta_col='metadata'): + if not data_path.exists(): + raise FileNotFoundError(f"File {data_path} not found.") + data = read_data(data_path) + content = data_path.read_text() + if isinstance(data, pd.DataFrame): + validate_cols(content_col, data) + return cls(data=data, content=content, content_col=content_col, meta_col=meta_col) + + def _get_docs_and_metadatas_by_df(self) -> (list, list): + df = self.data + docs = [] + metadatas = [] + for i in tqdm(range(len(df))): + docs.append(df[self.content_col].iloc[i]) + if self.meta_col: + metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) + else: + metadatas.append({}) + return docs, metadatas + + def _get_docs_and_metadatas_by_langchain(self) -> (list, list): + data = self.data + docs = [i.page_content for i in data] + metadatas = [i.metadata for i in data] + return docs, metadatas + + def get_docs_and_metadatas(self) -> (list, list): + if isinstance(self.data, pd.DataFrame): + return self._get_docs_and_metadatas_by_df() + elif isinstance(self.data, list): + return self._get_docs_and_metadatas_by_langchain() + else: + raise NotImplementedError("Data type not supported for metadata extraction.") + + +class Repo(BaseModel): + + # Name of this repo. + name: str = Field(default="") + docs: dict[Path, Document] = Field(default_factory=dict) + codes: dict[Path, Document] = Field(default_factory=dict) + assets: dict[Path, Document] = Field(default_factory=dict) + repo_path: Path = Field(default_factory=Path) + + def _path(self, filename): + return self.repo_path / filename + + @classmethod + def from_path(cls, repo_path: Path): + """Load documents, code, and assets from a repository path.""" + repo_path.mkdir(parents=True, exist_ok=True) + repo = Repo(repo_path = repo_path) + for file_path in repo_path.rglob('*'): + if file_path.is_file(): + repo._set(file_path.read_text(), file_path) + return repo + + def to_path(self): + """Persist all documents, code, and assets to the given repository path.""" + for doc in self.docs.values(): + doc.to_path() + for code in self.codes.values(): + code.to_path() + for asset in self.assets.values(): + asset.to_path() + + def _set(self, content: str, file_path: Path): + """Add a document to the appropriate category based on its file extension.""" + file_ext = file_path.suffix + + doc = Document(content=content, file_path=file_path) + if file_ext.lower() == '.md': + self.docs[file_path] = doc + elif file_ext.lower() in ['.py', '.js', '.css', '.html']: + self.codes[file_path] = doc + else: + self.assets[file_path] = doc + return doc + + def set(self, content: str, filename: str): + """Set a document and persist it to disk.""" + file_path = self._path(filename) + doc = self._set(content, file_path) + doc.to_path() + + def get(self, filename: str) -> Optional[Document]: + """Get a document by its filename.""" + path = self._path(filename) + return self.docs.get(path) or self.codes.get(path) or self.assets.get(path) + + +def main(): + repo1 = Repo.from_path(Path("/Users/alexanderwu/workspace/t1")) + repo1.set("wtf content", "doc/wtf_file.md") + repo1.set("wtf code", "code/wtf_file.py") + logger.info(repo1) # check doc + + +if __name__ == '__main__': + main() diff --git a/metagpt/document_store/base_store.py b/metagpt/document_store/base_store.py index 5d7015e8b..84b47a98c 100644 --- a/metagpt/document_store/base_store.py +++ b/metagpt/document_store/base_store.py @@ -28,20 +28,20 @@ class BaseStore(ABC): class LocalStore(BaseStore, ABC): - def __init__(self, raw_data: Path, cache_dir: Path = None): - if not raw_data: + def __init__(self, raw_data_path: Path, cache_dir: Path = None): + if not raw_data_path: raise FileNotFoundError self.config = Config() - self.raw_data = raw_data + self.raw_data_path = raw_data_path if not cache_dir: - cache_dir = raw_data.parent + cache_dir = raw_data_path.parent self.cache_dir = cache_dir self.store = self._load() if not self.store: self.store = self.write() def _get_index_and_store_fname(self): - fname = self.raw_data.name.split('.')[0] + fname = self.raw_data_path.name.split('.')[0] index_file = self.cache_dir / f"{fname}.index" store_file = self.cache_dir / f"{fname}.pkl" return index_file, store_file diff --git a/metagpt/document_store/document.py b/metagpt/document_store/document.py deleted file mode 100644 index e4b9473c7..000000000 --- a/metagpt/document_store/document.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/6/8 14:03 -@Author : alexanderwu -@File : document.py -""" -from pathlib import Path - -import pandas as pd -from langchain.document_loaders import ( - TextLoader, - UnstructuredPDFLoader, - UnstructuredWordDocumentLoader, -) -from langchain.text_splitter import CharacterTextSplitter -from tqdm import tqdm - - -def validate_cols(content_col: str, df: pd.DataFrame): - if content_col not in df.columns: - raise ValueError - - -def read_data(data_path: Path): - suffix = data_path.suffix - if '.xlsx' == suffix: - data = pd.read_excel(data_path) - elif '.csv' == suffix: - data = pd.read_csv(data_path) - elif '.json' == suffix: - data = pd.read_json(data_path) - elif suffix in ('.docx', '.doc'): - data = UnstructuredWordDocumentLoader(str(data_path), mode='elements').load() - elif '.txt' == suffix: - data = TextLoader(str(data_path)).load() - text_splitter = CharacterTextSplitter(separator='\n', chunk_size=256, chunk_overlap=0) - texts = text_splitter.split_documents(data) - data = texts - elif '.pdf' == suffix: - data = UnstructuredPDFLoader(str(data_path), mode="elements").load() - else: - raise NotImplementedError - return data - - -class Document: - - def __init__(self, data_path, content_col='content', meta_col='metadata'): - self.data = read_data(data_path) - if isinstance(self.data, pd.DataFrame): - validate_cols(content_col, self.data) - self.content_col = content_col - self.meta_col = meta_col - - def _get_docs_and_metadatas_by_df(self) -> (list, list): - df = self.data - docs = [] - metadatas = [] - for i in tqdm(range(len(df))): - docs.append(df[self.content_col].iloc[i]) - if self.meta_col: - metadatas.append({self.meta_col: df[self.meta_col].iloc[i]}) - else: - metadatas.append({}) - - return docs, metadatas - - def _get_docs_and_metadatas_by_langchain(self) -> (list, list): - data = self.data - docs = [i.page_content for i in data] - metadatas = [i.metadata for i in data] - return docs, metadatas - - def get_docs_and_metadatas(self) -> (list, list): - if isinstance(self.data, pd.DataFrame): - return self._get_docs_and_metadatas_by_df() - elif isinstance(self.data, list): - return self._get_docs_and_metadatas_by_langchain() - else: - raise NotImplementedError - \ No newline at end of file diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index dd450010d..885ad3e15 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -15,15 +15,15 @@ from langchain.vectorstores import FAISS from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore -from metagpt.document_store.document import Document +from metagpt.document import IndexableDocument from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data_path: Path, cache_dir=None, meta_col='source', content_col='output'): self.meta_col = meta_col self.content_col = content_col - super().__init__(raw_data, cache_dir) + super().__init__(raw_data_path, cache_dir) def _load(self) -> Optional["FaissStore"]: index_file, store_file = self._get_index_and_store_fname() @@ -60,9 +60,9 @@ class FaissStore(LocalStore): def write(self): """Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user.""" - if not self.raw_data.exists(): + if not self.raw_data_path.exists(): raise FileNotFoundError - doc = Document(self.raw_data, self.content_col, self.meta_col) + doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col) docs, metadatas = doc.get_docs_and_metadatas() self.store = self._write(docs, metadatas) diff --git a/metagpt/document_store/repo_parser.py b/metagpt/document_store/repo_parser.py new file mode 100644 index 000000000..f7e2b0f4a --- /dev/null +++ b/metagpt/document_store/repo_parser.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/17 17:58 +@Author : alexanderwu +@File : repo_parser.py +""" +import json +import pathlib +import ast + +import pandas as pd + + +class RepoParser: + def __init__(self): + self.base_directory = None + + def parse_file(self, file_path): + """Parse a Python file in the repository.""" + try: + return ast.parse(file_path.read_text()).body + except: + return [] + + def extract_class_and_function_info(self, tree, file_path): + """Extract class, function, and global variable information from the AST.""" + file_info = { + "file": str(file_path.relative_to(self.base_directory)), + "classes": [], + "functions": [], + "globals": [] + } + + for node in tree: + if isinstance(node, ast.ClassDef): + class_methods = [m.name for m in node.body if is_func(m)] + file_info["classes"].append({"name": node.name, "methods": class_methods}) + elif is_func(node): + file_info["functions"].append(node.name) + elif isinstance(node, ast.Assign) or isinstance(node, ast.AnnAssign): + for target in node.targets if isinstance(node, ast.Assign) else [node.target]: + if isinstance(target, ast.Name): + file_info["globals"].append(target.id) + return file_info + + def generate_json_structure(self, directory, output_path): + """Generate a JSON file documenting the repository structure.""" + files_classes = [] + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + output_path.write_text(json.dumps(files_classes, indent=4)) + + def generate_dataframe_structure(self, directory, output_path): + """Generate a DataFrame documenting the repository structure and save as CSV.""" + files_classes = [] + for path in directory.rglob('*.py'): + tree = self.parse_file(path) + file_info = self.extract_class_and_function_info(tree, path) + files_classes.append(file_info) + + df = pd.DataFrame(files_classes) + df.to_csv(output_path, index=False) + + def generate_structure(self, directory_path, output_path=None, mode='json'): + """Generate the structure of the repository as a specified format.""" + self.base_directory = pathlib.Path(directory_path) + output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}" + output_path = pathlib.Path(output_path) if output_path else output_file + + if mode == 'json': + self.generate_json_structure(self.base_directory, output_path) + elif mode == 'csv': + self.generate_dataframe_structure(self.base_directory, output_path) + + +def is_func(node): + return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + + +def main(): + repo_parser = RepoParser() + repo_parser.generate_structure("/Users/alexanderwu/git/mg1/metagpt", "/Users/alexanderwu/git/mg1/mg1-structure.csv", mode='csv') + + +if __name__ == '__main__': + main() diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..38077c90d 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -10,20 +10,22 @@ from typing import Iterable from pydantic import BaseModel, Field +# from metagpt.document import Document +from metagpt.document import Repo from metagpt.memory import Memory from metagpt.roles import Role from metagpt.schema import Message class Environment(BaseModel): - """环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到 - Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles - + """ + Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles """ roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) history: str = Field(default='') + repo: Repo = Field(default_factory=Repo) class Config: arbitrary_types_allowed = True @@ -50,6 +52,10 @@ class Environment(BaseModel): self.memory.add(message) self.history += f"\n{message}" + def publish_doc(self, content: str, filename: str): + """向当前环境发布文档（包括代码）""" + self.repo.set(content, filename) + async def run(self, k=1): """处理一次所有信息的运行 Process all Role runs at once diff --git a/metagpt/logs.py b/metagpt/logs.py index b2052e9b8..afebbfed9 100644 --- a/metagpt/logs.py +++ b/metagpt/logs.py @@ -10,15 +10,15 @@ import sys from loguru import logger as _logger -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT + def define_log_level(print_level="INFO", logfile_level="DEBUG"): - """调整日志级别到level之上 - Adjust the log level to above level - """ + """Adjust the log level to above level""" _logger.remove() _logger.add(sys.stderr, level=print_level) - _logger.add(PROJECT_ROOT / 'logs/log.txt', level=logfile_level) + _logger.add(METAGPT_ROOT / 'logs/log.txt', level=logfile_level) return _logger + logger = define_log_level() diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..7cbbe651e 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -14,7 +14,7 @@ class Manager: def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { - "BOSS": "Product Manager", + "User": "Product Manager", "Product Manager": "Architect", "Architect": "Engineer", "Engineer": "QA Engineer", diff --git a/metagpt/memory/__init__.py b/metagpt/memory/__init__.py index 710930626..bd6e72163 100644 --- a/metagpt/memory/__init__.py +++ b/metagpt/memory/__init__.py @@ -7,10 +7,10 @@ """ from metagpt.memory.memory import Memory -from metagpt.memory.longterm_memory import LongTermMemory +# from metagpt.memory.longterm_memory import LongTermMemory __all__ = [ "Memory", - "LongTermMemory", + # "LongTermMemory", ] diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 1f6685b38..171af47f0 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,7 +11,8 @@ from collections import OrderedDict from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.SummarizeCode import SummarizeCode +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -80,13 +81,13 @@ class Engineer(Role): self.n_borg = n_borg @classmethod - def parse_tasks(self, task_msg: Message) -> list[str]: + def parse_tasks(cls, task_msg: Message) -> list[str]: if task_msg.instruct_content: return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) @classmethod - def parse_code(self, code_text: str) -> str: + def parse_code(cls, code_text: str) -> str: return CodeParser.parse_code(block="", text=code_text) @classmethod @@ -98,10 +99,10 @@ class Engineer(Role): def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def recreate_workspace(self): workspace = self.get_workspace() @@ -167,7 +168,7 @@ class Engineer(Role): ) return msg - async def _act_sp_precision(self) -> Message: + async def _act_sp_with_cr(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: """ @@ -191,7 +192,6 @@ class Engineer(Role): code = rewrite_code except Exception as e: logger.error("code review failed!", e) - pass file_path = self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -199,6 +199,13 @@ class Engineer(Role): code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) + context = [] + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + code_review_all = await SummarizeCode().run(context=context_str) + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" @@ -209,5 +216,5 @@ class Engineer(Role): """Determines the mode of action based on whether code review is used.""" logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: - return await self._act_sp_precision() + return await self._act_sp_with_cr() return await self._act_sp() diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..f6172b607 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : product_manager.py """ -from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions import UserRequirement, WritePRD from metagpt.roles import Role @@ -38,4 +38,4 @@ class ProductManager(Role): """ super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..f124646b3 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -16,7 +16,8 @@ from metagpt.actions import ( WriteDesign, WriteTest, ) -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -50,13 +51,13 @@ class QaEngineer(Role): def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace_path / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace_path / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace_path / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b96c361c0..d772c0748 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -17,7 +17,8 @@ from metagpt.config import CONFIG from metagpt.actions import Action, ActionOutput from metagpt.llm import LLM, HumanProvider from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import Memory +# from metagpt.memory import LongTermMemory from metagpt.schema import Message PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -78,7 +79,7 @@ class RoleContext(BaseModel): """Role Runtime Context""" env: 'Environment' = Field(default=None) memory: Memory = Field(default_factory=Memory) - long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) + # long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) diff --git a/metagpt/roles/sk_agent.py b/metagpt/roles/sk_agent.py index b27841d74..4069f4836 100644 --- a/metagpt/roles/sk_agent.py +++ b/metagpt/roles/sk_agent.py @@ -9,7 +9,7 @@ from semantic_kernel.planning import SequentialPlanner from semantic_kernel.planning.action_planner.action_planner import ActionPlanner from semantic_kernel.planning.basic_planner import BasicPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions.execute_task import ExecuteTask from metagpt.logs import logger from metagpt.roles import Role @@ -39,7 +39,7 @@ class SkAgent(Role): """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([ExecuteTask()]) - self._watch([BossRequirement]) + self._watch([UserRequirement]) self.kernel = make_sk_kernel() # how funny the interface is inconsistent diff --git a/metagpt/software_company.py b/metagpt/software_company.py deleted file mode 100644 index d44a0068a..000000000 --- a/metagpt/software_company.py +++ /dev/null @@ -1,13 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/12 00:30 -@Author : alexanderwu -@File : software_company.py -""" -from metagpt.team import Team as SoftwareCompany - -import warnings -warnings.warn("metagpt.software_company is deprecated and will be removed in the future" - "Please use metagpt.team instead. SoftwareCompany class is now named as Team.", - DeprecationWarning, 2) diff --git a/metagpt/startup.py b/metagpt/startup.py new file mode 100644 index 000000000..d8ca4072f --- /dev/null +++ b/metagpt/startup.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from pathlib import Path +import asyncio +import typer + +app = typer.Typer() + + +@app.command() +def startup( + idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"), + investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."), + n_round: int = typer.Option(5, help="Number of rounds for the simulation."), + code_review: bool = typer.Option(True, help="Whether to use code review."), + run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."), + implement: bool = typer.Option(True, help="Enable or disable code implementation."), + project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'"), +): + """Run a startup. Be a boss.""" + from metagpt.roles import ProductManager, Architect, ProjectManager, Engineer, QaEngineer + from metagpt.team import Team + + company = Team() + company.hire( + [ + ProductManager(), + Architect(), + ProjectManager(), + ] + ) + + if implement or code_review: + company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + + if run_tests: + company.hire([QaEngineer()]) + + company.invest(investment) + company.start_project(project_name, idea) + asyncio.run(company.run(n_round=n_round)) + + +if __name__ == "__main__": + app() diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..2332aaa46 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -7,7 +7,7 @@ """ from pydantic import BaseModel, Field -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger @@ -21,7 +21,7 @@ class Team(BaseModel): Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging, dedicated to perform any multi-agent activity, such as collaboratively writing executable code. """ - environment: Environment = Field(default_factory=Environment) + env: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") @@ -30,7 +30,7 @@ class Team(BaseModel): def hire(self, roles: list[Role]): """Hire roles to cooperate""" - self.environment.add_roles(roles) + self.env.add_roles(roles) def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" @@ -42,10 +42,12 @@ class Team(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea, send_to: str = ""): - """Start a project from publishing boss requirement.""" + def start_project(self, project_name, idea, send_to: str = ""): + """Start a project from publishing user requirement.""" self.idea = idea - self.environment.publish_message(Message(role="Human", content=idea, cause_by=BossRequirement, send_to=send_to)) + # If user set project_name, then use it. + self.env.repo.name = project_name + self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to)) def _save(self): logger.info(self.json()) @@ -57,6 +59,6 @@ class Team(BaseModel): n_round -= 1 logger.debug(f"{n_round=}") self._check_balance() - await self.environment.run() - return self.environment.history + await self.env.run() + return self.env.history \ No newline at end of file diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index 1d9cd0b2a..4f010a912 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -13,12 +13,10 @@ from typing import List from aiohttp import ClientSession from PIL import Image, PngImagePlugin -from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +# from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger -config = Config() - payload = { "prompt": "", "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", @@ -56,9 +54,8 @@ default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution" class SDEngine: def __init__(self): # Initialize the SDEngine with configuration - self.config = Config() - self.sd_url = self.config.get("SD_URL") - self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}" + self.sd_url = CONFIG.get("SD_URL") + self.sd_t2i_url = f"{self.sd_url}{CONFIG.get('SD_T2I_API')}" # Define default payload settings for SD API self.payload = payload logger.info(self.sd_t2i_url) @@ -81,7 +78,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.workspace_path / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) @@ -125,6 +122,7 @@ def batch_decode_base64_to_image(imgs, save_dir="", save_name=""): save_name = join(save_dir, save_name) decode_base64_to_image(_img, save_name=save_name) + if __name__ == "__main__": engine = SDEngine() prompt = "pixel style, game design, a game interface should be minimalistic and intuitive with the score and high score displayed at the top. The snake and its food should be easily distinguishable. The game should have a simple color scheme, with a contrasting color for the snake and its food. Complete interface boundary" diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 204c22c67..eb85a3f90 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -10,7 +10,7 @@ import os from pathlib import Path from metagpt.config import CONFIG -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists @@ -69,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, if stdout: logger.info(stdout.decode()) if stderr: - logger.error(stderr.decode()) + logger.warning(stderr.decode()) else: if engine == "playwright": from metagpt.utils.mmdc_playwright import mermaid_to_file @@ -141,6 +141,6 @@ MMC2 = """sequenceDiagram if __name__ == "__main__": loop = asyncio.new_event_loop() - result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) - result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) + result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1")) loop.close() diff --git a/metagpt/utils/token_counter.py b/metagpt/utils/token_counter.py index 1af96f272..33bcd01a5 100644 --- a/metagpt/utils/token_counter.py +++ b/metagpt/utils/token_counter.py @@ -21,6 +21,7 @@ TOKEN_COSTS = { "gpt-4-32k": {"prompt": 0.06, "completion": 0.12}, "gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12}, "gpt-4-0613": {"prompt": 0.06, "completion": 0.12}, + "gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03}, "text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0}, "chatglm_turbo": {"prompt": 0.0, "completion": 0.00069} # 32k version, prompt + completion tokens=0.005￥/k-tokens } @@ -37,6 +38,7 @@ TOKEN_MAX = { "gpt-4-32k": 32768, "gpt-4-32k-0314": 32768, "gpt-4-0613": 8192, + "gpt-4-1106-preview": 128000, "text-embedding-ada-002": 8192, "chatglm_turbo": 32768 } @@ -56,16 +58,17 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"): "gpt-4-32k-0314", "gpt-4-0613", "gpt-4-32k-0613", + "gpt-4-1106-preview", }: tokens_per_message = 3 tokens_per_name = 1 elif model == "gpt-3.5-turbo-0301": tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n tokens_per_name = -1 # if there's a name, the role is omitted - elif "gpt-3.5-turbo" in model: + elif "gpt-3.5-turbo" == model: print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.") return count_message_tokens(messages, model="gpt-3.5-turbo-0613") - elif "gpt-4" in model: + elif "gpt-4" == model: print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.") return count_message_tokens(messages, model="gpt-4-0613") else: diff --git a/setup.py b/setup.py index 239156ae3..e7462767f 100644 --- a/setup.py +++ b/setup.py @@ -31,14 +31,14 @@ with open(path.join(here, "requirements.txt"), encoding="utf-8") as f: setup( name="metagpt", version="0.3.0", - description="The Multi-Role Meta Programming Framework", + description="The Multi-Agent Framework", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/geekan/MetaGPT", author="Alexander Wu", author_email="alexanderwu@fuzhi.ai", - license="Apache 2.0", - keywords="metagpt multi-role multi-agent programming gpt llm", + license="MIT", + keywords="metagpt multi-role multi-agent programming gpt llm metaprogramming", packages=find_packages(exclude=["contrib", "docs", "examples", "tests*"]), python_requires=">=3.9", install_requires=requirements, @@ -52,4 +52,9 @@ setup( cmdclass={ "install_mermaid": InstallMermaidCLI, }, + entry_points={ + 'console_scripts': [ + 'metagpt=metagpt.startup:app', + ], + }, ) diff --git a/startup.py b/startup.py deleted file mode 100644 index e9fbf94d3..000000000 --- a/startup.py +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -import asyncio - -import fire - -from metagpt.roles import ( - Architect, - Engineer, - ProductManager, - ProjectManager, - QaEngineer, -) -from metagpt.team import Team - - -async def startup( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = False, - run_tests: bool = False, - implement: bool = True, -): - """Run a startup. Be a boss.""" - company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) - - # if implement or code_review - if implement or code_review: - # developing features: implement the idea - company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - - if run_tests: - # developing features: run tests on the spot and identify bugs - # (bug fixing capability comes soon!) - company.hire([QaEngineer()]) - - company.invest(investment) - company.start_project(idea) - await company.run(n_round=n_round) - - -def main( - idea: str, - investment: float = 3.0, - n_round: int = 5, - code_review: bool = True, - run_tests: bool = False, - implement: bool = True, -): - """ - We are a software startup comprised of AI. By investing in us, - you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." - :param investment: As an investor, you have the opportunity to contribute - a certain dollar amount to this AI company. - :param n_round: - :param code_review: Whether to use code review. - :return: - """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) - - -if __name__ == "__main__": - fire.Fire(main) diff --git a/tests/metagpt/actions/mock.py b/tests/metagpt/actions/mock.py index a800690e8..5be1d8001 100644 --- a/tests/metagpt/actions/mock.py +++ b/tests/metagpt/actions/mock.py @@ -100,7 +100,7 @@ For testing, we can use the PyTest framework. This is a mature full-featured Pyt file_list = ["main.py", "room.py", "player.py", "game.py", "object.py", "puzzle.py", "test_game.py"] ``` -## Data structures and interface definitions: +## Data structures and interfaces: ```mermaid classDiagram class Room{ @@ -209,7 +209,7 @@ Shared knowledge for this project includes understanding the basic principles of """ ``` -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs. ```python """ The original requirements did not specify whether the game should have a save/load feature, multiplayer support, or any specific graphical user interface. More information on these aspects could help in further refining the product design and requirements. diff --git a/tests/metagpt/actions/test_write_prd.py b/tests/metagpt/actions/test_write_prd.py index 38e4e5221..18675ecc3 100644 --- a/tests/metagpt/actions/test_write_prd.py +++ b/tests/metagpt/actions/test_write_prd.py @@ -7,7 +7,7 @@ """ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.logs import logger from metagpt.roles.product_manager import ProductManager from metagpt.schema import Message @@ -17,7 +17,7 @@ from metagpt.schema import Message async def test_write_prd(): product_manager = ProductManager() requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结" - prd = await product_manager.handle(Message(content=requirements, cause_by=BossRequirement)) + prd = await product_manager.handle(Message(content=requirements, cause_by=UserRequirement)) logger.info(requirements) logger.info(prd) diff --git a/tests/metagpt/document_store/test_document.py b/tests/metagpt/document_store/test_document.py index 5ae357fb1..13c0921a3 100644 --- a/tests/metagpt/document_store/test_document.py +++ b/tests/metagpt/document_store/test_document.py @@ -7,22 +7,22 @@ """ import pytest -from metagpt.const import DATA_PATH -from metagpt.document_store.document import Document +from metagpt.const import METAGPT_ROOT +from metagpt.document import IndexableDocument CASES = [ - ("st/faq.xlsx", "Question", "Answer", 1), - ("cases/faq.csv", "Question", "Answer", 1), + ("requirements.txt", None, None, 0), + # ("cases/faq.csv", "Question", "Answer", 1), # ("cases/faq.json", "Question", "Answer", 1), - ("docx/faq.docx", None, None, 1), - ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 - ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 + # ("docx/faq.docx", None, None, 1), + # ("cases/faq.pdf", None, None, 0), # 这是因为pdf默认没有分割段落 + # ("cases/faq.txt", None, None, 0), # 这是因为txt按照256分割段落 ] @pytest.mark.parametrize("relative_path, content_col, meta_col, threshold", CASES) def test_document(relative_path, content_col, meta_col, threshold): - doc = Document(DATA_PATH / relative_path, content_col, meta_col) + doc = IndexableDocument.from_path(METAGPT_ROOT / relative_path, content_col, meta_col) rsp = doc.get_docs_and_metadatas() assert len(rsp[0]) > threshold assert len(rsp[1]) > threshold diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index dc5540520..ac9362937 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -4,7 +4,7 @@ from metagpt.config import CONFIG from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.role import RoleContext from metagpt.memory import LongTermMemory @@ -15,24 +15,24 @@ def test_ltm_search(): assert len(openai_api_key) > 20 role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + rc = RoleContext(watch=[UserRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) news = ltm.find_news([message]) assert len(news) == 1 ltm.add(message) sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) news = ltm.find_news([sim_message]) assert len(news) == 0 ltm.add(sim_message) new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm.find_news([new_message]) assert len(news) == 1 ltm.add(new_message) @@ -48,7 +48,7 @@ def test_ltm_search(): assert len(news) == 0 new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) news = ltm_new.find_news([new_message]) assert len(news) == 1 diff --git a/tests/metagpt/memory/test_memory_storage.py b/tests/metagpt/memory/test_memory_storage.py index 6bb3e8f1d..bd4441641 100644 --- a/tests/metagpt/memory/test_memory_storage.py +++ b/tests/metagpt/memory/test_memory_storage.py @@ -6,7 +6,7 @@ from typing import List from metagpt.memory.memory_storage import MemoryStorage from metagpt.schema import Message -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.actions import WritePRD from metagpt.actions.action_output import ActionOutput @@ -14,7 +14,7 @@ from metagpt.actions.action_output import ActionOutput def test_idea_message(): idea = 'Write a cli snake game' role_id = 'UTUser1(Product Manager)' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + message = Message(role='User', content=idea, cause_by=UserRequirement) memory_storage: MemoryStorage = MemoryStorage() messages = memory_storage.recover_memory(role_id) @@ -24,12 +24,12 @@ def test_idea_message(): assert memory_storage.is_initialized is True sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_message = Message(role='User', content=sim_idea, cause_by=UserRequirement) new_messages = memory_storage.search(sim_message) assert len(new_messages) == 0 # similar, return [] new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_message = Message(role='User', content=new_idea, cause_by=UserRequirement) new_messages = memory_storage.search(new_message) assert new_messages[0].content == message.content @@ -49,7 +49,7 @@ def test_actionout_message(): ic_obj = ActionOutput.create_model_class('prd', out_mapping) role_id = 'UTUser2(Architect)' - content = 'The boss has requested the creation of a command-line interface (CLI) snake game' + content = 'The user has requested the creation of a command-line interface (CLI) snake game' message = Message(content=content, instruct_content=ic_obj(**out_data), role='user', diff --git a/tests/metagpt/planner/test_action_planner.py b/tests/metagpt/planner/test_action_planner.py index 5ab9a493f..8efe6cfc4 100644 --- a/tests/metagpt/planner/test_action_planner.py +++ b/tests/metagpt/planner/test_action_planner.py @@ -9,7 +9,7 @@ import pytest from semantic_kernel.core_skills import FileIOSkill, MathSkill, TextSkill, TimeSkill from semantic_kernel.planning.action_planner.action_planner import ActionPlanner -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -23,7 +23,7 @@ async def test_action_planner(): role.import_skill(TimeSkill(), "time") role.import_skill(TextSkill(), "text") task = "What is the sum of 110 and 990?" - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # it will choose mathskill.Add assert "1100" == (await role._act()).content diff --git a/tests/metagpt/planner/test_basic_planner.py b/tests/metagpt/planner/test_basic_planner.py index 03a82ec5e..f6d44ba03 100644 --- a/tests/metagpt/planner/test_basic_planner.py +++ b/tests/metagpt/planner/test_basic_planner.py @@ -8,7 +8,7 @@ import pytest from semantic_kernel.core_skills import TextSkill -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.const import SKILL_DIRECTORY from metagpt.roles.sk_agent import SkAgent from metagpt.schema import Message @@ -26,7 +26,7 @@ async def test_basic_planner(): role.import_semantic_skill_from_directory(SKILL_DIRECTORY, "WriterSkill") role.import_skill(TextSkill(), "TextSkill") # using BasicPlanner - role.recv(Message(content=task, cause_by=BossRequirement)) + role.recv(Message(content=task, cause_by=UserRequirement)) await role._think() # assuming sk_agent will think he needs WriterSkill.Brainstorm and WriterSkill.Translate assert "WriterSkill.Brainstorm" in role.plan.generated_plan.result diff --git a/tests/metagpt/roles/mock.py b/tests/metagpt/roles/mock.py index 52fc4a3c1..fbad06acb 100644 --- a/tests/metagpt/roles/mock.py +++ b/tests/metagpt/roles/mock.py @@ -5,10 +5,10 @@ @Author : alexanderwu @File : mock.py """ -from metagpt.actions import BossRequirement, WriteDesign, WritePRD, WriteTasks +from metagpt.actions import UserRequirement, WriteDesign, WritePRD, WriteTasks from metagpt.schema import Message -BOSS_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" +USER_REQUIREMENT = """开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结""" DETAIL_REQUIREMENT = """需求：开发一个基于LLM（大语言模型）与私有知识库的搜索引擎，希望有几点能力 1. 用户可以在私有知识库进行搜索，再根据大语言模型进行总结，输出的结果包括了总结 @@ -94,7 +94,7 @@ SYSTEM_DESIGN = '''## Python package name ] ``` -## Data structures and interface definitions +## Data structures and interfaces ```mermaid classDiagram class Main { @@ -252,7 +252,7 @@ a = 'a' class MockMessages: - req = Message(role="Boss", content=BOSS_REQUIREMENT, cause_by=BossRequirement) + req = Message(role="User", content=USER_REQUIREMENT, cause_by=UserRequirement) prd = Message(role="Product Manager", content=PRD, cause_by=WritePRD) system_design = Message(role="Architect", content=SYSTEM_DESIGN, cause_by=WriteDesign) tasks = Message(role="Project Manager", content=TASKS, cause_by=WriteTasks) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..102c6ebd6 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,8 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +# from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -29,7 +30,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Selected Elements:Provide as Plain text, up to 5 specified elements, clear and simple ## HTML Layout:Provide as Plain text, use standard HTML code ## CSS Styles (styles.css):Provide as Plain text,use standard css code -## Anything UNCLEAR:Provide as Plain text. Make clear here. +## Anything UNCLEAR:Provide as Plain text. Try to clarify it. """ @@ -214,7 +215,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace_path / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..b27bc3da7 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -8,7 +8,7 @@ import pytest -from metagpt.actions import BossRequirement +from metagpt.actions import UserRequirement from metagpt.environment import Environment from metagpt.logs import logger from metagpt.manager import Manager @@ -49,7 +49,7 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) env.set_manager(Manager()) - env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) + env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) await env.run(k=2) logger.info(f"{env.history=}") diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..fea58bc29 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,9 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine + def test_sd_engine_init(): @@ -21,5 +23,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace_path / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True diff --git a/tests/metagpt/utils/test_common.py b/tests/metagpt/utils/test_common.py index ec4443175..b6c000f9b 100644 --- a/tests/metagpt/utils/test_common.py +++ b/tests/metagpt/utils/test_common.py @@ -10,7 +10,7 @@ import os import pytest -from metagpt.const import get_project_root +from metagpt.const import get_metagpt_root class TestGetProjectRoot: @@ -20,11 +20,11 @@ class TestGetProjectRoot: os.chdir(abs_root) def test_get_project_root(self): - project_root = get_project_root() + project_root = get_metagpt_root() assert project_root.name == 'metagpt' def test_get_root_exception(self): with pytest.raises(Exception) as exc_info: self.change_etc_dir() - get_project_root() + get_metagpt_root() assert str(exc_info.value) == "Project root not found." diff --git a/tests/metagpt/utils/test_output_parser.py b/tests/metagpt/utils/test_output_parser.py index 4e362f9f7..99ab1f79e 100644 --- a/tests/metagpt/utils/test_output_parser.py +++ b/tests/metagpt/utils/test_output_parser.py @@ -218,7 +218,7 @@ We need clarification on how the high score should be stored. Should it persist } t_text1 = '''## Original Requirements: -The boss wants to create a web-based version of the game "Fly Bird". +The user wants to create a web-based version of the game "Fly Bird". ## Product Goals: diff --git a/tests/metagpt/utils/test_read_docx.py b/tests/metagpt/utils/test_read_docx.py index a7d0774a8..adf473ae7 100644 --- a/tests/metagpt/utils/test_read_docx.py +++ b/tests/metagpt/utils/test_read_docx.py @@ -6,12 +6,12 @@ @File : test_read_docx.py """ -from metagpt.const import PROJECT_ROOT +from metagpt.const import METAGPT_ROOT from metagpt.utils.read_document import read_docx class TestReadDocx: def test_read_docx(self): - docx_sample = PROJECT_ROOT / "tests/data/docx_for_test.docx" + docx_sample = METAGPT_ROOT / "tests/data/docx_for_test.docx" docx = read_docx(docx_sample) assert len(docx) == 6