diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 5ed31bed8..b027616f7 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -70,6 +70,6 @@ class DebugError(Action): prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=rsp) + code = CodeParser.parse_code(text=rsp) return code diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py index 711e56d39..548555196 100644 --- a/metagpt/actions/di/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -30,7 +30,7 @@ class WriteAnalysisCode(Action): ) rsp = await self._aask(reflection_prompt, system_msgs=[REFLECTION_SYSTEM_MSG]) - reflection = json.loads(CodeParser.parse_code(block=None, text=rsp)) + reflection = json.loads(CodeParser.parse_code(text=rsp)) return reflection["improved_impl"] @@ -57,7 +57,7 @@ class WriteAnalysisCode(Action): code = await self._debug_with_reflection(context=context, working_memory=working_memory) else: rsp = await self.llm.aask(context, system_msgs=[INTERPRETER_SYSTEM_MSG], **kwargs) - code = CodeParser.parse_code(block=None, text=rsp) + code = CodeParser.parse_code(text=rsp) return code @@ -69,5 +69,5 @@ class CheckData(Action): code_written = "\n\n".join(code_written) prompt = CHECK_DATA_PROMPT.format(code_written=code_written) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block=None, text=rsp) + code = CodeParser.parse_code(text=rsp) return code diff --git a/metagpt/actions/di/write_plan.py b/metagpt/actions/di/write_plan.py index 8d6eccf57..efea9f526 100644 --- a/metagpt/actions/di/write_plan.py +++ b/metagpt/actions/di/write_plan.py @@ -47,7 +47,7 @@ class WritePlan(Action): context="\n".join([str(ct) for ct in context]), max_tasks=max_tasks, task_type_desc=task_type_desc ) rsp = await self._aask(prompt) - rsp = CodeParser.parse_code(block=None, text=rsp) + rsp = CodeParser.parse_code(text=rsp) return rsp diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 08f2c2fcb..bba3deaa3 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -9,10 +9,12 @@ """ import shutil from pathlib import Path -from typing import Optional +from typing import Dict, Optional -from metagpt.actions import Action, ActionOutput +from metagpt.actions import Action, UserRequirement from metagpt.const import REQUIREMENT_FILENAME +from metagpt.schema import AIMessage +from metagpt.utils.common import any_to_str from metagpt.utils.file_repository import FileRepository @@ -21,6 +23,13 @@ class PrepareDocuments(Action): name: str = "PrepareDocuments" i_context: Optional[str] = None + key_descriptions: Optional[Dict[str, str]] = None + send_to: str + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if not self.key_descriptions: + self.key_descriptions = {"project_path": 'the project path if exists in "Original Requirement"'} @property def config(self): @@ -40,10 +49,26 @@ class PrepareDocuments(Action): async def run(self, with_messages, **kwargs): """Create and initialize the workspace folder, initialize the Git environment.""" + user_requirements = [i for i in with_messages if i.cause_by == any_to_str(UserRequirement)] + if not self.config.project_path and user_requirements and self.key_descriptions: + args = await user_requirements[0].parse_resources(llm=self.llm, key_descriptions=self.key_descriptions) + for k, v in args.items(): + if not v or k in ["resources", "reason"]: + continue + self.context.kwargs.set(k, v) + if self.context.kwargs.project_path: + self.config.update_via_cli( + project_path=self.context.kwargs.project_path, + project_name="", + inc=False, + reqa_file=self.context.kwargs.reqa_file or "", + max_auto_summarize_code=0, + ) + self._init_repo() # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content) # Send a Message notification to the WritePRD action, instructing it to process requirements using # `docs/requirement.txt` and `docs/prd/`. - return ActionOutput(content=doc.content, instruct_content=doc) + return AIMessage(content="", instruct_content=doc, cause_by=self, send_to=self.send_to) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 0942bac1e..6cfde2385 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -88,7 +88,7 @@ class WriteCode(Action): @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=code_rsp) + code = CodeParser.parse_code(text=code_rsp) return code async def run(self, *args, **kwargs) -> CodingContext: diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index ac6fe7045..ed868c867 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -132,7 +132,7 @@ class WriteCodeReview(Action): # if LBTM, rewrite code rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" code_rsp = await self._aask(rewrite_prompt) - code = CodeParser.parse_code(block="", text=code_rsp) + code = CodeParser.parse_code(text=code_rsp) return result, code async def run(self, *args, **kwargs) -> CodingContext: diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 978fa20a6..286d3ea13 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -45,7 +45,7 @@ class WriteTest(Action): code_rsp = await self._aask(prompt) try: - code = CodeParser.parse_code(block="", text=code_rsp) + code = CodeParser.parse_code(text=code_rsp) except Exception: # Handle the exception if needed logger.error(f"Can't parse the code: {code_rsp}") diff --git a/metagpt/const.py b/metagpt/const.py index 979f00cf4..eaa22434f 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -145,3 +145,6 @@ MARKDOWN_TITLE_PREFIX = "## " # Reporter METAGPT_REPORTER_DEFAULT_URL = os.environ.get("METAGPT_REPORTER_URL", "") + +# Metadata defines +AGENT = "agent" diff --git a/metagpt/environment/mgx/mgx_env.py b/metagpt/environment/mgx/mgx_env.py index 6c7069b56..c4ed8c4f1 100644 --- a/metagpt/environment/mgx/mgx_env.py +++ b/metagpt/environment/mgx/mgx_env.py @@ -6,6 +6,7 @@ from metagpt.actions import ( WriteTest, ) from metagpt.actions.summarize_code import SummarizeCode +from metagpt.const import AGENT from metagpt.environment.base_env import Environment from metagpt.logs import get_human_input from metagpt.roles import ( @@ -23,6 +24,9 @@ from metagpt.utils.common import any_to_str, any_to_str_set class MGXEnv(Environment): """MGX Environment""" + # Before enabling TL to fully take over the routing, all software company roles need to be able to handle TL messages, which requires restructuring. + allow_bypass_team_leader: bool = True + def _publish_message(self, message: Message, peekable: bool = True) -> bool: return super().publish_message(message, peekable) @@ -35,7 +39,11 @@ class MGXEnv(Environment): # bypass team leader, team leader only needs to know but not to react tl.rc.memory.add(self.move_message_info_to_content(message)) - elif self.message_within_software_sop(message) and not self.has_user_requirement(): + elif ( + self.allow_bypass_team_leader + and self.message_within_software_sop(message) + and not self.has_user_requirement() + ): # Quick routing for messages within software SOP, bypassing TL. # Use rules to check for user intervention and to finish task. # NOTE: This escapes TL's supervision and has pitfalls such as routing obsolete messages even if TL has acquired a new user requirement. @@ -47,6 +55,9 @@ class MGXEnv(Environment): tl.finish_current_task() elif publicer == tl.profile: + if message.send_to == {"no one"}: + # skip the dummy message from team leader + return True # message processed by team leader can be published now self._publish_message(message) @@ -71,7 +82,7 @@ class MGXEnv(Environment): def message_within_software_sop(self, message: Message) -> bool: return message.sent_from in any_to_str_set([ProductManager, Architect, ProjectManager, Engineer, QaEngineer]) - def has_user_requirement(self, k=2) -> bool: + def has_user_requirement(self, k=1) -> bool: """A heuristics to check if there is a recent user intervention""" return any_to_str(UserRequirement) in [msg.cause_by for msg in self.history.get(k)] @@ -86,10 +97,8 @@ class MGXEnv(Environment): 1. Convert role, since role field must be reserved for LLM API, and is limited to, for example, one of ["user", "assistant", "system"] 2. Add sender and recipient info to content, making TL aware, since LLM API only takes content as input """ - if message.role in ["system", "user", "assistant"]: - sent_from = message.sent_from - else: - sent_from = message.role + if message.role not in ["system", "user", "assistant"]: message.role = "assistant" + sent_from = message.metadata[AGENT] if AGENT in message.metadata else message.sent_from message.content = f"from {sent_from} to {message.send_to}: {message.content}" return message diff --git a/metagpt/ext/stanford_town/roles/st_role.py b/metagpt/ext/stanford_town/roles/st_role.py index 79f58b07d..4856548f0 100644 --- a/metagpt/ext/stanford_town/roles/st_role.py +++ b/metagpt/ext/stanford_town/roles/st_role.py @@ -181,13 +181,13 @@ class STRole(Role): logger.info(f"Role: {self.name} saved role's memory into {str(self.role_storage_path)}") - async def _observe(self, ignore_memory=False) -> int: + async def _observe(self) -> int: if not self.rc.env: return 0 news = [] if not news: news = self.rc.msg_buffer.pop_all() - old_messages = [] if ignore_memory else self.rc.memory.get() + old_messages = [] if not self.enable_memory else self.rc.memory.get() # Filter out messages of interest. self.rc.news = [ n for n in news if (n.cause_by in self.rc.watch or self.name in n.send_to) and n not in old_messages diff --git a/metagpt/prompts/di/data_analyst.py b/metagpt/prompts/di/data_analyst.py new file mode 100644 index 000000000..f8e2c1cb5 --- /dev/null +++ b/metagpt/prompts/di/data_analyst.py @@ -0,0 +1,41 @@ +CMD_PROMPT = """ +# Data Structure +class Task(BaseModel): + task_id: str = "" + dependent_task_ids: list[str] = [] + instruction: str = "" + task_type: str = "" + assignee: str = "David" + +# Available Commands +{available_commands} + +# Current Plan +{plan_status} + +# Example +{example} + +# Instructions +Based on the context, write a plan or modify an existing plan to achieve the goal. A plan consists of one to 3 tasks. +If plan is created, you should track the progress and update the plan accordingly, such as finish_current_task, append_task, reset_task, replace_task, etc. +Pay close attention to new user message, review the conversation history, use reply_to_human to respond to new user requirement. +Note: +1. If you keeping encountering errors, unexpected situation, or you are not sure of proceeding, use ask_human to ask for help. +2. Each time you finish a task, use reply_to_human to report your progress. + +You may use any of the available commands to create a plan or update the plan. You may output mutiple commands, they will be executed sequentially. +If you finish current task, you will automatically take the next task in the existing plan, use finish_task, DON'T append a new task. + +# Your commands in a json array, in the following output format, always output a json array, if there is nothing to do, use the pass command: +Some text indicating your thoughts, such as how you should update the plan status, respond to inquiry, or seek for help. Then a json array of commands. +```json +[ + {{ + "command_name": str, + "args": {{"arg_name": arg_value, ...}} + }}, + ... +] +``` +""" diff --git a/metagpt/prompts/di/team_leader.py b/metagpt/prompts/di/team_leader.py index 87e8a1336..c5f6a79aa 100644 --- a/metagpt/prompts/di/team_leader.py +++ b/metagpt/prompts/di/team_leader.py @@ -1,12 +1,8 @@ -from metagpt.strategy.thinking_command import Command - - -def prepare_command_prompt(commands: list[Command]) -> str: - command_prompt = "" - for i, command in enumerate(commands): - command_prompt += f"{i+1}. {command.value.signature}:\n{command.value.desc}\n\n" - return command_prompt - +SYSTEM_PROMPT = """ +You are a team leader, and you are responsible for drafting tasks and routing tasks to your team members. +When drafting and routing tasks, ALWAYS include necessary or important info inside the instruction, such as path, link, environment to team members, because you are their sole info source. +Each time you do something, reply to human letting them know what you did. +""" CMD_PROMPT = """ # Data Structure diff --git a/metagpt/prompts/di/write_analysis_code.py b/metagpt/prompts/di/write_analysis_code.py index d2b4f1299..af941808d 100644 --- a/metagpt/prompts/di/write_analysis_code.py +++ b/metagpt/prompts/di/write_analysis_code.py @@ -1,7 +1,9 @@ INTERPRETER_SYSTEM_MSG = """ As a data scientist, you need to help user to achieve their goal step by step in a continuous Jupyter notebook. Since it is a notebook environment, don't use asyncio.run. Instead, use await if you need to call an async function. -If you want to use shell command such as git clone, pip install packages, navigate folders, read file, etc., use Terminal tool if available before trying ! in notebook block. +If you want to use shell command such as git clone, pip install packages, navigate folders, read file, etc., use Terminal tool if available. DON'T use ! in notebook block. +Don't write all codes in one response, each time, just write code for one step or current task. +While some concise thoughts are helpful, code is absolutely required. Always output one and only one code block in your response. """ STRUCTUAL_PROMPT = """ diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index dbfed72df..120c1d3cb 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -220,7 +220,7 @@ class OpenAILLM(BaseLLM): # The response content is `code``, but it appears in the content instead of the arguments. code_formats = "```" if message.content.startswith(code_formats) and message.content.endswith(code_formats): - code = CodeParser.parse_code(None, message.content) + code = CodeParser.parse_code(text=message.content) return {"language": "python", "code": code} # reponse is message return {"language": "markdown", "code": self.get_choice_text(rsp)} diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 166f8cfd0..465beff05 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -6,9 +6,11 @@ @File : architect.py """ -from metagpt.actions import WritePRD +from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.design_api import WriteDesign +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles.role import Role +from metagpt.utils.common import any_to_str class Architect(Role): @@ -32,8 +34,24 @@ class Architect(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + self.enable_memory = False # Initialize actions specific to the Architect role - self.set_actions([WriteDesign]) + self.set_actions([PrepareDocuments(send_to=any_to_str(self), context=self.context), WriteDesign]) # Set events or actions the Architect should watch or be aware of - self._watch({WritePRD}) + self._watch({UserRequirement, PrepareDocuments, WritePRD}) + + async def _think(self) -> bool: + """Decide what to do""" + mappings = { + any_to_str(UserRequirement): 0, + any_to_str(PrepareDocuments): 1, + any_to_str(WritePRD): 1, + } + for i in self.rc.news: + idx = mappings.get(i.cause_by, -1) + if idx < 0: + continue + self.rc.todo = self.actions[idx] + return bool(self.rc.todo) + return False diff --git a/metagpt/roles/di/data_analyst.py b/metagpt/roles/di/data_analyst.py new file mode 100644 index 000000000..4c959575b --- /dev/null +++ b/metagpt/roles/di/data_analyst.py @@ -0,0 +1,114 @@ +from __future__ import annotations + +import json +from typing import Literal + +from pydantic import model_validator + +from metagpt.actions import Action +from metagpt.actions.di.write_analysis_code import WriteAnalysisCode +from metagpt.logs import logger +from metagpt.prompts.di.data_analyst import CMD_PROMPT +from metagpt.roles.di.data_interpreter import DataInterpreter +from metagpt.schema import Message, TaskResult +from metagpt.strategy.planner import Planner +from metagpt.strategy.thinking_command import ( + Command, + prepare_command_prompt, + run_commands, +) +from metagpt.tools.tool_recommend import BM25ToolRecommender +from metagpt.utils.common import CodeParser + + +class DataAnalyst(DataInterpreter): + name: str = "David" + profile: str = "DataAnalyst" + react_mode: Literal["react"] = "react" + max_react_loop: int = 20 # used for react mode + task_result: TaskResult = None + available_commands: list[Command] = [ + Command.APPEND_TASK, + Command.RESET_TASK, + Command.REPLACE_TASK, + Command.FINISH_CURRENT_TASK, + # Command.PUBLISH_MESSAGE, + Command.ASK_HUMAN, + Command.REPLY_TO_HUMAN, + # Command.PASS, + ] + commands: list[dict] = [] # issued commands to be executed + + @model_validator(mode="after") + def set_plan_and_tool(self) -> "DataInterpreter": + # We force using this parameter for DataAnalyst + assert self.react_mode == "react" + assert self.auto_run + assert self.use_plan + + # Roughly the same part as DataInterpreter.set_plan_and_tool + self._set_react_mode(react_mode=self.react_mode, max_react_loop=self.max_react_loop, auto_run=self.auto_run) + if self.tools and not self.tool_recommender: + self.tool_recommender = BM25ToolRecommender(tools=self.tools) + self.set_actions([WriteAnalysisCode]) + self._set_state(0) + + # HACK: Init Planner, control it through dynamic thinking; Consider formalizing as a react mode + self.planner = Planner(goal=self.goal, working_memory=self.rc.working_memory, auto_run=True) + + return self + + async def _think(self) -> bool: + """Useful in 'react' mode. Use LLM to decide whether and what to do next.""" + self._set_state(0) + if not self.planner.plan.goal: + self.user_requirement = self.get_memories()[-1].content + self.planner.plan.goal = self.user_requirement + else: + self.working_memory.add_batch(self.rc.news) + + plan_status = self.planner.plan.model_dump(include=["goal", "tasks"]) + for task in plan_status["tasks"]: + task.pop("code") + task.pop("result") + example = "" + prompt = CMD_PROMPT.format( + plan_status=plan_status, + example=example, + available_commands=prepare_command_prompt(self.available_commands), + ) + context = self.llm.format_msg(self.working_memory.get() + [Message(content=prompt, role="user")]) + + rsp = await self.llm.aask(context) + self.commands = json.loads(CodeParser.parse_code(block=None, text=rsp)) + self.rc.memory.add(Message(content=rsp, role="assistant")) + + await run_commands(self, self.commands, self.rc.working_memory) + + return bool(self.rc.todo) + + async def _act(self) -> Message: + """Useful in 'react' mode. Return a Message conforming to Role._act interface.""" + logger.info(f"ready to take on task {self.planner.plan.current_task}") + code, result, is_success = await self._write_and_exec_code() + self.planner.plan.current_task.is_success = ( + is_success # mark is_success, determine is_finished later in thinking + ) + self.task_result = TaskResult(code=code, result=result, is_success=is_success) + return Message(content="Task completed", role="assistant", sent_from=self._setting, cause_by=WriteAnalysisCode) + + async def _react(self) -> Message: + actions_taken = 0 + rsp = Message(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act + while actions_taken < self.rc.max_react_loop: + # NOTE: difference here, keep observing within react + await self._observe() + # think + has_todo = await self._think() + if not has_todo: + break + # act + logger.debug(f"{self._setting}: {self.rc.state=}, will do {self.rc.todo}") + rsp = await self._act() + actions_taken += 1 + return rsp # return output from the last action diff --git a/metagpt/roles/di/data_interpreter.py b/metagpt/roles/di/data_interpreter.py index 2e1e0a2da..e147cbbe3 100644 --- a/metagpt/roles/di/data_interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -5,7 +5,7 @@ from typing import Literal from pydantic import Field, model_validator -from metagpt.actions.di.ask_review import ReviewConst +# from metagpt.actions.di.ask_review import ReviewConst from metagpt.actions.di.execute_nb_code import ExecuteNbCode from metagpt.actions.di.write_analysis_code import CheckData, WriteAnalysisCode from metagpt.logs import logger @@ -43,6 +43,7 @@ class DataInterpreter(Role): tool_recommender: ToolRecommender = None react_mode: Literal["plan_and_act", "react"] = "plan_and_act" max_react_loop: int = 10 # used for react mode + user_requirement: str = "" @model_validator(mode="after") def set_plan_and_tool(self) -> "Interpreter": @@ -62,7 +63,7 @@ class DataInterpreter(Role): async def _think(self) -> bool: """Useful in 'react' mode. Use LLM to decide whether and what to do next.""" - user_requirement = self.get_memories()[-1].content + self.user_requirement = self.get_memories()[-1].content context = self.working_memory.get() if not context: @@ -71,9 +72,9 @@ class DataInterpreter(Role): self._set_state(0) return True - prompt = REACT_THINK_PROMPT.format(user_requirement=user_requirement, context=context) + prompt = REACT_THINK_PROMPT.format(user_requirement=self.user_requirement, context=context) rsp = await self.llm.aask(prompt) - rsp_dict = json.loads(CodeParser.parse_code(block=None, text=rsp)) + rsp_dict = json.loads(CodeParser.parse_code(text=rsp)) self.working_memory.add(Message(content=rsp_dict["thoughts"], role="assistant")) need_action = rsp_dict["state"] self._set_state(0) if need_action else self._set_state(-1) @@ -83,7 +84,7 @@ class DataInterpreter(Role): async def _act(self) -> Message: """Useful in 'react' mode. Return a Message conforming to Role._act interface.""" code, _, _ = await self._write_and_exec_code() - return Message(content=code, role="assistant", cause_by=WriteAnalysisCode) + return Message(content=code, role="assistant", sent_from=self._setting, cause_by=WriteAnalysisCode) async def _plan_and_act(self) -> Message: self._set_state(0) @@ -136,11 +137,11 @@ class DataInterpreter(Role): ### process execution result ### counter += 1 - if not success and counter >= max_retry: - logger.info("coding failed!") - review, _ = await self.planner.ask_review(auto_run=False, trigger=ReviewConst.CODE_REVIEW_TRIGGER) - if ReviewConst.CHANGE_WORDS[0] in review: - counter = 0 # redo the task again with help of human suggestions + # if not success and counter >= max_retry: + # logger.info("coding failed!") + # review, _ = await self.planner.ask_review(auto_run=False, trigger=ReviewConst.CODE_REVIEW_TRIGGER) + # if ReviewConst.CHANGE_WORDS[0] in review: + # counter = 0 # redo the task again with help of human suggestions return code, result, success @@ -154,10 +155,8 @@ class DataInterpreter(Role): logger.info(f"ready to {todo.name}") use_reflection = counter > 0 and self.use_reflection # only use reflection after the first trial - user_requirement = self.get_memories()[-1].content - code = await todo.run( - user_requirement=user_requirement, + user_requirement=self.user_requirement, plan_status=plan_status, tool_info=tool_info, working_memory=self.working_memory.get(), diff --git a/metagpt/roles/di/team_leader.py b/metagpt/roles/di/team_leader.py index 9f951799a..a1ef11fa6 100644 --- a/metagpt/roles/di/team_leader.py +++ b/metagpt/roles/di/team_leader.py @@ -5,17 +5,20 @@ import json from pydantic import model_validator from metagpt.actions.di.run_command import RunCommand -from metagpt.environment.mgx.mgx_env import MGXEnv from metagpt.prompts.di.team_leader import ( CMD_PROMPT, FINISH_CURRENT_TASK_CMD, - prepare_command_prompt, + SYSTEM_PROMPT, ) from metagpt.roles import Role -from metagpt.schema import Message, Task, TaskResult +from metagpt.schema import Message, TaskResult from metagpt.strategy.experience_retriever import SimpleExpRetriever from metagpt.strategy.planner import Planner -from metagpt.strategy.thinking_command import Command +from metagpt.strategy.thinking_command import ( + Command, + prepare_command_prompt, + run_commands, +) from metagpt.utils.common import CodeParser @@ -23,7 +26,7 @@ class TeamLeader(Role): name: str = "Tim" profile: str = "Team Leader" task_result: TaskResult = None - commands: list[Command] = [ + available_commands: list[Command] = [ Command.APPEND_TASK, Command.RESET_TASK, Command.REPLACE_TASK, @@ -33,49 +36,15 @@ class TeamLeader(Role): Command.REPLY_TO_HUMAN, Command.PASS, ] + commands: list[dict] = [] # issued commands to be executed @model_validator(mode="after") def set_plan(self) -> "TeamLeader": self.planner = Planner(goal=self.goal, working_memory=self.rc.working_memory, auto_run=True) return self - def _run_env_command(self, cmd): - assert isinstance(self.rc.env, MGXEnv), "TeamLeader should only be used in an MGXEnv" - if cmd["command_name"] == Command.PUBLISH_MESSAGE.cmd_name: - self.publish_message(Message(**cmd["args"])) - elif cmd["command_name"] == Command.ASK_HUMAN.cmd_name: - self.rc.env.ask_human(sent_from=self, **cmd["args"]) - elif cmd["command_name"] == Command.REPLY_TO_HUMAN.cmd_name: - self.rc.env.reply_to_human(sent_from=self, **cmd["args"]) - - def _run_internal_command(self, cmd): - if cmd["command_name"] == Command.APPEND_TASK.cmd_name: - self.planner.plan.append_task(Task(**cmd["args"])) - elif cmd["command_name"] == Command.RESET_TASK.cmd_name: - self.planner.plan.reset_task(**cmd["args"]) - elif cmd["command_name"] == Command.REPLACE_TASK.cmd_name: - self.planner.plan.replace_task(Task(**cmd["args"])) - elif cmd["command_name"] == Command.FINISH_CURRENT_TASK.cmd_name: - self.planner.plan.current_task.update_task_result(task_result=self.task_result) - self.planner.plan.finish_current_task() - self.rc.working_memory.clear() - - def run_commands(self, cmds): - print(*cmds, sep="\n") - for cmd in cmds: - self._run_env_command(cmd) - self._run_internal_command(cmd) - - if self.planner.plan.is_plan_finished(): - self._set_state(-1) - - def get_memory(self, k=10) -> list[Message]: - """A wrapper with default value""" - return self.rc.memory.get(k=k) - async def _think(self) -> bool: """Useful in 'react' mode. Use LLM to decide whether and what to do next.""" - self.commands = [] if not self.planner.plan.goal: user_requirement = self.get_memories()[-1].content @@ -96,20 +65,19 @@ class TeamLeader(Role): plan_status=plan_status, team_info=team_info, example=example, - available_commands=prepare_command_prompt(self.commands), + available_commands=prepare_command_prompt(self.available_commands), ) - context = self.llm.format_msg(self.get_memory() + [Message(content=prompt, role="user")]) + context = self.llm.format_msg(self.get_memories(k=10) + [Message(content=prompt, role="user")]) - rsp = await self.llm.aask(context) - rsp_dict = json.loads(CodeParser.parse_code(block=None, text=rsp)) - self.commands.extend(rsp_dict) + rsp = await self.llm.aask(context, system_msgs=[SYSTEM_PROMPT]) + self.commands = json.loads(CodeParser.parse_code(text=rsp)) self.rc.memory.add(Message(content=rsp, role="assistant")) return True async def _act(self) -> Message: """Useful in 'react' mode. Return a Message conforming to Role._act interface.""" - self.run_commands(self.commands) + await run_commands(self, self.commands, self.rc.memory) self.task_result = TaskResult(result="Success", is_success=True) msg = Message(content="Commands executed", send_to="no one") # a dummy message to conform to the interface self.rc.memory.add(msg) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 30fb6f187..e7b5cf219 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -24,8 +24,15 @@ from collections import defaultdict from pathlib import Path from typing import List, Optional, Set -from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.actions import ( + Action, + UserRequirement, + WriteCode, + WriteCodeReview, + WriteTasks, +) from metagpt.actions.fix_bug import FixBug +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST from metagpt.actions.summarize_code import SummarizeCode from metagpt.actions.write_code_plan_and_change_an import WriteCodePlanAndChange @@ -39,6 +46,7 @@ from metagpt.const import ( from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( + AIMessage, CodePlanAndChangeContext, CodeSummarizeContext, CodingContext, @@ -53,6 +61,7 @@ from metagpt.utils.common import ( get_project_srcs_path, init_python_folder, ) +from metagpt.utils.git_repository import ChangeType IS_PASS_PROMPT = """ {context} @@ -93,9 +102,20 @@ class Engineer(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - + self.enable_memory = False self.set_actions([WriteCode]) - self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug, WriteCodePlanAndChange]) + self._watch( + [ + UserRequirement, + PrepareDocuments, + WriteTasks, + SummarizeCode, + WriteCode, + WriteCodeReview, + FixBug, + WriteCodePlanAndChange, + ] + ) self.code_todos = [] self.summarize_todos = [] self.next_todo_action = any_to_name(WriteCode) @@ -130,13 +150,11 @@ class Engineer(Role): dependencies=list(dependencies), content=coding_context.code_doc.content, ) - msg = Message( + AIMessage( content=coding_context.model_dump_json(), instruct_content=coding_context, - role=self.profile, cause_by=WriteCode, ) - self.rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) if not changed_files: @@ -156,13 +174,12 @@ class Engineer(Role): if isinstance(self.rc.todo, SummarizeCode): self.next_todo_action = any_to_name(WriteCode) return await self._act_summarize() - return None + return await self.rc.todo.run(self.rc.history) async def _act_write_code(self): changed_files = await self._act_sp_with_cr(review=self.use_code_review) - return Message( + return AIMessage( content="\n".join(changed_files), - role=self.profile, cause_by=WriteCodeReview if self.use_code_review else WriteCode, send_to=self, sent_from=self, @@ -195,9 +212,8 @@ class Engineer(Role): logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}") if not tasks or self.config.max_auto_summarize_code == 0: - return Message( + return AIMessage( content="", - role=self.profile, cause_by=SummarizeCode, sent_from=self, send_to="Edward", # The name of QaEngineer @@ -205,9 +221,7 @@ class Engineer(Role): # The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. # This parameter is used for debugging the workflow. self.n_summarize += 1 if self.config.max_auto_summarize_code > self.n_summarize else 0 - return Message( - content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self - ) + return AIMessage(content=json.dumps(tasks), cause_by=SummarizeCode, send_to=self, sent_from=self) async def _act_code_plan_and_change(self): """Write code plan and change that guides subsequent WriteCode and WriteCodeReview""" @@ -229,9 +243,8 @@ class Engineer(Role): dependencies=dependencies, ) - return Message( + return AIMessage( content=code_plan_and_change, - role=self.profile, cause_by=WriteCodePlanAndChange, send_to=self, sent_from=self, @@ -245,14 +258,25 @@ class Engineer(Role): return False, rsp async def _think(self) -> Action | None: - if not self.src_workspace: - self.src_workspace = get_project_srcs_path(self.project_repo.workdir) - write_plan_and_change_filters = any_to_str_set([WriteTasks, FixBug]) - write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode]) - summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if not self.rc.news: return None msg = self.rc.news[0] + if msg.cause_by == any_to_str(UserRequirement): + self.rc.todo = PrepareDocuments( + key_descriptions={ + "project_path": 'the project path if exists in "Original Requirement"', + "src_filename": 'the file name of the source code file explicitly requested for modification if exists in "Original Requirement"', + }, + context=self.context, + send_to=any_to_str(self), + ) + return self.rc.todo + + if not self.src_workspace: + self.src_workspace = get_project_srcs_path(self.project_repo.workdir) + write_plan_and_change_filters = any_to_str_set([PrepareDocuments, WriteTasks, FixBug]) + write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if self.config.inc and msg.cause_by in write_plan_and_change_filters: logger.debug(f"TODO WriteCodePlanAndChange:{msg.model_dump_json()}") await self._new_code_plan_and_change_action(cause_by=msg.cause_by) @@ -308,7 +332,11 @@ class Engineer(Role): async def _new_code_actions(self): bug_fix = await self._is_fixbug() # Prepare file repos - changed_src_files = self.project_repo.srcs.all_files if bug_fix else self.project_repo.srcs.changed_files + changed_src_files = self.project_repo.srcs.changed_files + if self.context.kwargs.src_filename: + changed_src_files = {self.context.kwargs.src_filename: ChangeType.UNTRACTED} + if bug_fix: + changed_src_files = self.project_repo.srcs.all_files changed_task_files = self.project_repo.docs.task.changed_files changed_files = Documents() # Recode caused by upstream changes. @@ -319,6 +347,8 @@ class Engineer(Role): task_list = self._parse_tasks(task_doc) await self._init_python_folder(task_list) for task_filename in task_list: + if self.context.kwargs.src_filename and task_filename != self.context.kwargs.src_filename: + continue old_code_doc = await self.project_repo.srcs.get(task_filename) if not old_code_doc: old_code_doc = Document( diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 4eb1d249a..4beab5366 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -10,7 +10,7 @@ from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles.role import Role, RoleReactMode -from metagpt.utils.common import any_to_name +from metagpt.utils.common import any_to_name, any_to_str class ProductManager(Role): @@ -32,8 +32,8 @@ class ProductManager(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - - self.set_actions([PrepareDocuments, WritePRD]) + self.enable_memory = False + self.set_actions([PrepareDocuments(send_to=any_to_str(self)), WritePRD]) self._watch([UserRequirement, PrepareDocuments]) self.rc.react_mode = RoleReactMode.BY_ORDER self.todo_action = any_to_name(WritePRD) @@ -47,6 +47,3 @@ class ProductManager(Role): self.config.git_reinit = False self.todo_action = any_to_name(WritePRD) return bool(self.rc.todo) - - async def _observe(self, ignore_memory=False) -> int: - return await super()._observe(ignore_memory=True) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 422d2889b..70bd3bf8b 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -6,9 +6,11 @@ @File : project_manager.py """ -from metagpt.actions import WriteTasks +from metagpt.actions import UserRequirement, WriteTasks from metagpt.actions.design_api import WriteDesign +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles.role import Role +from metagpt.utils.common import any_to_str class ProjectManager(Role): @@ -32,6 +34,21 @@ class ProjectManager(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + self.enable_memory = False + self.set_actions([PrepareDocuments(send_to=any_to_str(self), context=self.context), WriteTasks]) + self._watch([UserRequirement, PrepareDocuments, WriteDesign]) - self.set_actions([WriteTasks]) - self._watch([WriteDesign]) + async def _think(self) -> bool: + """Decide what to do""" + mappings = { + any_to_str(UserRequirement): 0, + any_to_str(PrepareDocuments): 1, + any_to_str(WriteDesign): 1, + } + for i in self.rc.news: + idx = mappings.get(i.cause_by, -1) + if idx < 0: + continue + self.rc.todo = self.actions[idx] + return bool(self.rc.todo) + return False diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 04440c1cb..ed9c455a6 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -15,13 +15,19 @@ of SummarizeCode. """ -from metagpt.actions import DebugError, RunCode, WriteTest +from metagpt.actions import DebugError, RunCode, UserRequirement, WriteTest +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.actions.summarize_code import SummarizeCode from metagpt.const import MESSAGE_ROUTE_TO_NONE from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Document, Message, RunCodeContext, TestingContext -from metagpt.utils.common import any_to_str_set, init_python_folder, parse_recipient +from metagpt.schema import AIMessage, Document, Message, RunCodeContext, TestingContext +from metagpt.utils.common import ( + any_to_str, + any_to_str_set, + init_python_folder, + parse_recipient, +) class QaEngineer(Role): @@ -37,19 +43,22 @@ class QaEngineer(Role): def __init__(self, **kwargs): super().__init__(**kwargs) + self.enable_memory = False # FIXME: a bit hack here, only init one action to circumvent _think() logic, # will overwrite _think() in future updates - self.set_actions([WriteTest]) - self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) + self.set_actions( + [ + WriteTest, + ] + ) + self._watch([UserRequirement, PrepareDocuments, SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 async def _write_test(self, message: Message) -> None: src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs - changed_files = set(src_file_repo.changed_files.keys()) - # Unit tests only. - if self.config.reqa_file and self.config.reqa_file not in changed_files: - changed_files.add(self.config.reqa_file) + reqa_file = self.context.kwargs.reqa_file or self.config.reqa_file + changed_files = {reqa_file} if reqa_file else set(src_file_repo.changed_files.keys()) for filename in changed_files: # write tests if not filename or "test" in filename: @@ -80,9 +89,8 @@ class QaEngineer(Role): additional_python_paths=[str(self.context.src_workspace)], ) self.publish_message( - Message( + AIMessage( content=run_code_context.model_dump_json(), - role=self.profile, cause_by=WriteTest, sent_from=self, send_to=self, @@ -116,9 +124,8 @@ class QaEngineer(Role): recipient = parse_recipient(result.summary) mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} self.publish_message( - Message( + AIMessage( content=run_code_context.model_dump_json(), - role=self.profile, cause_by=RunCode, sent_from=self, send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), @@ -131,9 +138,8 @@ class QaEngineer(Role): await self.project_repo.tests.save(filename=run_code_context.test_filename, content=code) run_code_context.output = None self.publish_message( - Message( + AIMessage( content=run_code_context.model_dump_json(), - role=self.profile, cause_by=DebugError, sent_from=self, send_to=self, @@ -141,18 +147,18 @@ class QaEngineer(Role): ) async def _act(self) -> Message: - await init_python_folder(self.project_repo.tests.workdir) + if self.project_path: + await init_python_folder(self.project_repo.tests.workdir) if self.test_round > self.test_round_allowed: - result_msg = Message( + result_msg = AIMessage( content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", - role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg - code_filters = any_to_str_set({SummarizeCode}) + code_filters = any_to_str_set({PrepareDocuments, SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self.rc.news: @@ -167,16 +173,26 @@ class QaEngineer(Role): elif msg.cause_by in run_filters: # I ran my test code, time to fix bugs, if any await self._debug_error(msg) + elif msg.cause_by == any_to_str(UserRequirement): + return await self._parse_user_requirement(msg) self.test_round += 1 - return Message( + return AIMessage( content=f"Round {self.test_round} of tests done", - role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) - async def _observe(self, ignore_memory=False) -> int: - # This role has events that trigger and execute themselves based on conditions, and cannot rely on the - # content of memory to activate. - return await super()._observe(ignore_memory=True) + async def _parse_user_requirement(self, msg: Message) -> AIMessage: + action = PrepareDocuments( + send_to=any_to_str(self), + key_descriptions={ + "project_path": 'the project path if exists in "Original Requirement"', + "reqa_file": 'the file name to rewrite unit test if exists in "Original Requirement"', + }, + context=self.context, + ) + rsp = await action.run([msg]) + if not self.src_workspace: + self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name + return rsp diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index abd6fbb61..80a4d6744 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -34,7 +34,14 @@ from metagpt.context_mixin import ContextMixin from metagpt.logs import logger from metagpt.memory import Memory from metagpt.provider import HumanProvider -from metagpt.schema import Message, MessageQueue, SerializationMixin +from metagpt.schema import ( + AIMessage, + Message, + MessageQueue, + SerializationMixin, + Task, + TaskResult, +) from metagpt.strategy.planner import Planner from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator from metagpt.utils.project_repo import ProjectRepo @@ -134,6 +141,9 @@ class Role(SerializationMixin, ContextMixin, BaseModel): constraints: str = "" desc: str = "" is_human: bool = False + enable_memory: bool = ( + True # Stateless, atomic roles, or roles that use external storage can disable this to save memory. + ) role_id: str = "" states: list[str] = [] @@ -245,10 +255,9 @@ class Role(SerializationMixin, ContextMixin, BaseModel): return self def _init_action(self, action: Action): - if not action.private_config: - action.set_llm(self.llm, override=True) - else: - action.set_llm(self.llm, override=False) + action.set_context(self.context) + override = not action.private_config + action.set_llm(self.llm, override=override) action.set_prefix(self._get_prefix()) def set_action(self, action: Action): @@ -390,22 +399,22 @@ class Role(SerializationMixin, ContextMixin, BaseModel): logger.info(f"{self._setting}: to do {self.rc.todo}({self.rc.todo.name})") response = await self.rc.todo.run(self.rc.history) if isinstance(response, (ActionOutput, ActionNode)): - msg = Message( + msg = AIMessage( content=response.content, instruct_content=response.instruct_content, - role=self._setting, cause_by=self.rc.todo, sent_from=self, ) elif isinstance(response, Message): msg = response else: - msg = Message(content=response, role=self.profile, cause_by=self.rc.todo, sent_from=self) - self.rc.memory.add(msg) + msg = AIMessage(content=response, cause_by=self.rc.todo, sent_from=self) + if self.enable_memory: + self.rc.memory.add(msg) return msg - async def _observe(self, ignore_memory=False) -> int: + async def _observe(self) -> int: """Prepare new messages for processing from the message buffer and other sources.""" # Read unprocessed messages from the msg buffer. news = [] @@ -414,12 +423,12 @@ class Role(SerializationMixin, ContextMixin, BaseModel): if not news: news = self.rc.msg_buffer.pop_all() # Store the read messages in your own memory to prevent duplicate processing. - old_messages = [] if ignore_memory else self.rc.memory.get() - self.rc.memory.add_batch(news) - # Filter out messages of interest. + old_messages = [] if not self.enable_memory else self.rc.memory.get() + # Filter in messages of interest. self.rc.news = [ n for n in news if (n.cause_by in self.rc.watch or self.name in n.send_to) and n not in old_messages ] + self.rc.memory.add_batch(self.rc.news) # only save messages of interest into memory self.latest_observed_msg = self.rc.news[-1] if self.rc.news else None # record the latest observed msg # Design Rules: @@ -451,7 +460,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel): Use llm to select actions in _think dynamically """ actions_taken = 0 - rsp = Message(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act + rsp = AIMessage(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act while actions_taken < self.rc.max_react_loop: # think has_todo = await self._think() @@ -466,7 +475,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel): async def _act_by_order(self) -> Message: """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" start_idx = self.rc.state if self.rc.state >= 0 else 0 # action to run from recovered state - rsp = Message(content="No actions taken yet") # return default message if actions=[] + rsp = AIMessage(content="No actions taken yet") # return default message if actions=[] for i in range(start_idx, len(self.states)): self._set_state(i) rsp = await self._act() @@ -523,6 +532,8 @@ class Role(SerializationMixin, ContextMixin, BaseModel): else: raise ValueError(f"Unsupported react mode: {self.rc.react_mode}") self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + if isinstance(rsp, AIMessage): + rsp.with_agent(self._setting) return rsp def get_memories(self, k=0) -> list[Message]: diff --git a/metagpt/schema.py b/metagpt/schema.py index 3746c74f2..cfe991cd9 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -38,6 +38,7 @@ from pydantic import ( ) from metagpt.const import ( + AGENT, MESSAGE_ROUTE_CAUSE_BY, MESSAGE_ROUTE_FROM, MESSAGE_ROUTE_TO, @@ -48,7 +49,7 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.repo_parser import DotClassInfo -from metagpt.utils.common import any_to_str, any_to_str_set, import_class +from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, import_class from metagpt.utils.exceptions import handle_exception from metagpt.utils.report import TaskReporter from metagpt.utils.serialize import ( @@ -187,6 +188,14 @@ class Documents(BaseModel): return ActionOutput(content=self.model_dump_json(), instruct_content=self) +class Resource(BaseModel): + """Used by `Message`.`parse_resources`""" + + resource_type: str # the type of resource + value: str # a string type of resource content + description: str # explanation + + class Message(BaseModel): """list[: ]""" @@ -197,6 +206,7 @@ class Message(BaseModel): cause_by: str = Field(default="", validate_default=True) sent_from: str = Field(default="", validate_default=True) send_to: set[str] = Field(default={MESSAGE_ROUTE_TO_ALL}, validate_default=True) + metadata: Dict[str, str] = Field(default_factory=dict) # metadata for `content` and `instruct_content` @field_validator("id", mode="before") @classmethod @@ -312,14 +322,53 @@ class Message(BaseModel): logger.error(f"parse json failed: {val}, error:{err}") return None + async def parse_resources(self, llm: "BaseLLM", key_descriptions: Dict[str, str] = None) -> Dict: + """ + `parse_resources` corresponds to the in-context adaptation capability of the input of the atomic action, + which will be migrated to the context builder later. + + Args: + llm (BaseLLM): The instance of the BaseLLM class. + key_descriptions (Dict[str, str], optional): A dictionary containing descriptions for each key, + if provided. Defaults to None. + + Returns: + Dict: A dictionary containing parsed resources. + + """ + if not self.content: + return {} + content = f"## Original Requirement\n```text\n{self.content}\n```\n" + return_format = ( + "Return a markdown JSON object with:\n" + '- a "resources" key contain a list of objects. Each object with:\n' + ' - a "resource_type" key explain the type of resource;\n' + ' - a "value" key containing a string type of resource content;\n' + ' - a "description" key explaining why;\n' + ) + key_descriptions = key_descriptions or {} + for k, v in key_descriptions.items(): + return_format += f'- a "{k}" key containing {v};\n' + return_format += '- a "reason" key explaining why;\n' + instructions = ['Lists all the resources contained in the "Original Requirement".', return_format] + rsp = await llm.aask(msg=content, system_msgs=instructions) + json_data = CodeParser.parse_code(text=rsp, lang="json") + m = json.loads(json_data) + m["resources"] = [Resource(**i) for i in m.get("resources", [])] + return m + + def add_metadata(self, key: str, value: str): + self.metadata[key] = value + class UserMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ - def __init__(self, content: str): - super().__init__(content=content, role="user") + def __init__(self, content: str, **kwargs): + kwargs.pop("role", None) + super().__init__(content=content, role="user", **kwargs) class SystemMessage(Message): @@ -327,8 +376,9 @@ class SystemMessage(Message): Facilitate support for OpenAI messages """ - def __init__(self, content: str): - super().__init__(content=content, role="system") + def __init__(self, content: str, **kwargs): + kwargs.pop("role", None) + super().__init__(content=content, role="system", **kwargs) class AIMessage(Message): @@ -336,8 +386,17 @@ class AIMessage(Message): Facilitate support for OpenAI messages """ - def __init__(self, content: str): - super().__init__(content=content, role="assistant") + def __init__(self, content: str, **kwargs): + kwargs.pop("role", None) + super().__init__(content=content, role="assistant", **kwargs) + + def with_agent(self, name: str): + self.add_metadata(key=AGENT, value=name) + return self + + @property + def agent(self) -> str: + return self.metadata.get(AGENT, "") class Task(BaseModel): diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 17e299135..78e15ab47 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -7,6 +7,8 @@ from pathlib import Path import typer from metagpt.const import CONFIG_ROOT +from metagpt.utils.common import any_to_str +from metagpt.utils.project_repo import ProjectRepo app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False) @@ -64,7 +66,7 @@ def generate_repo( idea = company.idea company.invest(investment) - company.run_project(idea) + company.run_project(idea, send_to=any_to_str(ProductManager)) asyncio.run(company.run(n_round=n_round)) return ctx.repo diff --git a/metagpt/strategy/experience_retriever.py b/metagpt/strategy/experience_retriever.py index e0d2601e3..34ff184ab 100644 --- a/metagpt/strategy/experience_retriever.py +++ b/metagpt/strategy/experience_retriever.py @@ -68,6 +68,12 @@ class SimpleExpRetriever(ExpRetriever): "content": "User request to create a cli snake game. Please create a product requirement document (PRD) outlining the features, user interface, and user experience of the snake game.", "send_to": "Alice" } + }, + { + "command_name": "reply_to_human", + "args": { + "content": "I have assigned the tasks to the team members. Alice will create the PRD, Bob will design the software architecture, Eve will break down the architecture into tasks, Alex will implement the core game logic, and Edward will write comprehensive tests. The team will work on the project accordingly", + } } ] ``` @@ -92,6 +98,12 @@ class SimpleExpRetriever(ExpRetriever): "content": "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy.", "send_to": "David" } + }, + { + "command_name": "reply_to_human", + "args": { + "content": "I have assigned the task to David. He will break down the task further by himself and starts solving it.", + } } ] ``` @@ -110,10 +122,16 @@ class SimpleExpRetriever(ExpRetriever): "args": {} }, { - "command_name": "publish_message", + "command_name": "publish_message", + "args": { + "content": "Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.", + "send_to": "Bob" + } + }, + { + "command_name": "reply_to_human", "args": { - "content": "Please design the software architecture for the snake game based on the PRD created by Alice. The PRD is at 'docs/prd/20240424153821.json'. Include the choice of programming language, libraries, and data flow, etc.", - "send_to": "Bob" + "content": "Alice has completed the PRD. I have marked her task as finished and sent the PRD to Bob. Bob will work on the software architecture.", } } ] diff --git a/metagpt/strategy/thinking_command.py b/metagpt/strategy/thinking_command.py index 7b48cae51..f852c3764 100644 --- a/metagpt/strategy/thinking_command.py +++ b/metagpt/strategy/thinking_command.py @@ -1,7 +1,14 @@ +from __future__ import annotations + from enum import Enum from pydantic import BaseModel +from metagpt.environment.mgx.mgx_env import MGXEnv +from metagpt.memory import Memory +from metagpt.roles import Role +from metagpt.schema import Message, Task + class CommandDef(BaseModel): name: str @@ -59,3 +66,49 @@ class Command(Enum): @property def cmd_name(self): return self.value.name + + +def prepare_command_prompt(commands: list[Command]) -> str: + command_prompt = "" + for i, command in enumerate(commands): + command_prompt += f"{i+1}. {command.value.signature}:\n{command.value.desc}\n\n" + return command_prompt + + +async def run_env_command(role: Role, cmd: list[dict], role_memory: Memory = None): + assert isinstance(role.rc.env, MGXEnv), "TeamLeader should only be used in an MGXEnv" + if cmd["command_name"] == Command.PUBLISH_MESSAGE.cmd_name: + role.publish_message(Message(**cmd["args"])) + if cmd["command_name"] == Command.ASK_HUMAN.cmd_name: + # TODO: Operation on role memory should not appear here, consider moving it into role + role.rc.working_memory.add(Message(content=cmd["args"]["question"], role="assistant")) + human_rsp = await role.rc.env.ask_human(sent_from=role, **cmd["args"]) + role.rc.working_memory.add(Message(content=human_rsp, role="user")) + elif cmd["command_name"] == Command.REPLY_TO_HUMAN.cmd_name: + # TODO: consider if the message should go into memory + await role.rc.env.reply_to_human(sent_from=role, **cmd["args"]) + + +def run_plan_command(role: Role, cmd: list[dict]): + if cmd["command_name"] == Command.APPEND_TASK.cmd_name: + role.planner.plan.append_task(Task(**cmd["args"])) + elif cmd["command_name"] == Command.RESET_TASK.cmd_name: + role.planner.plan.reset_task(**cmd["args"]) + elif cmd["command_name"] == Command.REPLACE_TASK.cmd_name: + role.planner.plan.replace_task(Task(**cmd["args"])) + elif cmd["command_name"] == Command.FINISH_CURRENT_TASK.cmd_name: + if role.planner.plan.is_plan_finished(): + return + role.planner.plan.current_task.update_task_result(task_result=role.task_result) + role.planner.plan.finish_current_task() + role.rc.working_memory.clear() + + +async def run_commands(role: Role, cmds: list[dict], role_memory: Memory = None): + print(*cmds, sep="\n") + for cmd in cmds: + await run_env_command(role, cmd, role_memory) + run_plan_command(role, cmd) + + if role.planner.plan.is_plan_finished(): + role._set_state(-1) diff --git a/metagpt/strategy/tot.py b/metagpt/strategy/tot.py index 88c2ac9ff..17ce63211 100644 --- a/metagpt/strategy/tot.py +++ b/metagpt/strategy/tot.py @@ -62,7 +62,7 @@ class ThoughtSolverBase(BaseModel): current_state=current_state, **{"n_generate_sample": self.config.n_generate_sample} ) rsp = await self.llm.aask(msg=state_prompt + "\n" + OUTPUT_FORMAT) - thoughts = CodeParser.parse_code(block="", text=rsp) + thoughts = CodeParser.parse_code(text=rsp) thoughts = eval(thoughts) # fixme 避免不跟随，生成过多nodes # valid_thoughts = [_node for idx, _node in enumerate(thoughts) if idx < self.n_generate_sample] diff --git a/metagpt/team.py b/metagpt/team.py index 79c4c36aa..c3498b96b 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -20,7 +20,7 @@ from metagpt.context import Context from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message +from metagpt.schema import UserMessage from metagpt.utils.common import ( NoMoneyException, read_json_file, @@ -102,7 +102,7 @@ class Team(BaseModel): # Human requirement. self.env.publish_message( - Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL), + UserMessage(content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL), peekable=False, ) diff --git a/metagpt/tools/libs/__init__.py b/metagpt/tools/libs/__init__.py index d807a7d41..f4a112a69 100644 --- a/metagpt/tools/libs/__init__.py +++ b/metagpt/tools/libs/__init__.py @@ -16,15 +16,7 @@ from metagpt.tools.libs import ( browser, deployer, ) -from metagpt.tools.libs.software_development import ( - write_prd, - write_design, - write_project_plan, - write_codes, - run_qa_test, - fix_bug, - git_archive, -) +from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description _ = ( data_preprocess, @@ -33,15 +25,12 @@ _ = ( gpt_v_generator, web_scraping, email_login, - write_prd, - write_design, - write_project_plan, - write_codes, - run_qa_test, - fix_bug, - git_archive, terminal, file_manager, browser, deployer, + get_env, + get_env_description, + set_get_env_entry, + default_get_env, ) # Avoid pre-commit error diff --git a/metagpt/tools/libs/env.py b/metagpt/tools/libs/env.py new file mode 100644 index 000000000..fc4c314c5 --- /dev/null +++ b/metagpt/tools/libs/env.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/4/25 +@Author : mashenquan +@File : env.py +@Desc: Implement `get_env`. RFC 216 2.4.2.4.2. +""" +import os +from typing import Dict + + +class EnvKeyNotFoundError(Exception): + def __init__(self, info): + super().__init__(info) + + +async def default_get_env(key: str, app_name: str = None) -> str: + if key in os.environ: + return os.environ[key] + + from metagpt.context import Context + + context = Context() + val = context.kwargs.get(key, None) + if val is not None: + return val + + raise EnvKeyNotFoundError(f"EnvKeyNotFoundError: {key}, app_name:{app_name or ''}") + + +async def default_get_env_description() -> Dict[str, str]: + result = {} + for k in os.environ.keys(): + call = f'await get_env(key="{k}", app_name="")' + result[call] = f"Return the value of environment variable `{k}`." + + from metagpt.context import Context + + context = Context() + for k in context.kwargs.__dict__.keys(): + call = f'await get_env(key="{k}", app_name="")' + result[call] = f"Get the value of environment variable `{k}`." + return result + + +_get_env_entry = default_get_env +_get_env_description_entry = default_get_env_description + + +async def get_env(key: str, app_name: str = None) -> str: + """ + Retrieve the value of the environment variable for the specified key. + + Args: + key (str): The key of the environment variable. + app_name (str, optional): The name of the application. Defaults to None. + + Returns: + str: The value corresponding to the given key in the environment variables. + If no value is found for the given key, an empty string is returned. + + Example: + This function can be used to retrieve environment variables asynchronously. + It should be called using `await`. + + >>> from metagpt.tools.libs.env import get_env + >>> api_key = await get_env("API_KEY") + >>> print(api_key) + + + >>> from metagpt.tools.libs.env import get_env + >>> api_key = await get_env(key="API_KEY", app_name="GITHUB") + >>> print(api_key) + + + Note: + This is an asynchronous function and must be called using `await`. + """ + global _get_env_entry + if _get_env_entry: + return await _get_env_entry(key=key, app_name=app_name) + + return await default_get_env(key=key, app_name=app_name) + + +async def get_env_description() -> Dict[str, str]: + global _get_env_description_entry + + if _get_env_description_entry: + return await _get_env_description_entry() + + return await default_get_env_description() + + +def set_get_env_entry(value, description): + """Modify `get_env` entry and `get_description` entry. + + Args: + value (function): New function entry. + description (str): Description of the function. + + This function modifies the `get_env` entry by updating the function + to the provided `value` and its description to the provided `description`. + """ + global _get_env_entry + global _get_env_description_entry + _get_env_entry = value + _get_env_description_entry = description diff --git a/metagpt/tools/libs/git.py b/metagpt/tools/libs/git.py index afbcb8b0b..ac9e68bd8 100644 --- a/metagpt/tools/libs/git.py +++ b/metagpt/tools/libs/git.py @@ -3,6 +3,10 @@ from __future__ import annotations from pathlib import Path +from typing import Optional + +from github.Issue import Issue +from github.PullRequest import PullRequest from metagpt.tools.tool_registry import register_tool from metagpt.utils.git_repository import GitRepository @@ -63,3 +67,147 @@ async def git_checkout(repo_dir: str | Path, commit_id: str): if not repo.is_valid: ValueError(f"Invalid git root: {repo_dir}") await repo.checkout(commit_id) + + +@register_tool(tags=["git"]) +async def create_pull_request( + access_token: str, + base: str, + head: str, + base_repo_name: str, + head_repo_name: Optional[str] = None, + title: Optional[str] = None, + body: Optional[str] = None, +) -> PullRequest: + """ + Creates a pull request in a Git repository. + + Args: + access_token (str): The access token for authentication. + base (str): The name of the base branch of the pull request (e.g., 'main', 'master'). + head (str): The name of the head branch of the pull request (e.g., 'feature-branch'). + base_repo_name (str): The full repository name (user/repo) where the pull request will be created. + head_repo_name (Optional[str], optional): The full repository name (user/repo) where the pull request will merge from. Defaults to None. + title (Optional[str]): The title of the pull request. + body (Optional[str]): The body of the pull request. + + + Returns: + PullRequest: The created pull request object. + + Raises: + ValueError: If `access_token` is invalid. Visit: "https://github.com/settings/tokens" + Any exceptions that might occur during the pull request creation process. + + Note: + This function is intended to be used in an asynchronous context (with `await`). + + Example: + >>> # Merge Request + >>> repo_name = "user/repo" # "user/repo" for example: "https://github.com/user/repo.git" + >>> base = "master" # branch that merge to + >>> head = "feature/new_feature" # branch that merge from + >>> title = "Implement new feature" + >>> body = "This pull request adds functionality X, Y, and Z." + >>> pull_request = await create_pull_request( + repo_name=repo_name, + base=base, + head=head, + title=title, + body=body, + access_token=get_env("git_access_token") + ) + >>> print(pull_request) + PullRequest(title="Implement new feature", number=26) + + >>> # Pull Request + >>> base_repo_name = "user1/repo1" # for example: "user1/repo1" from "https://github.com/user1/repo1.git" + >>> head_repo_name = "user2/repo2" # for example: "user2/repo2" from "https://github.com/user2/repo2.git" + >>> base = "master" # branch that merge to + >>> head = "feature/new_feature" # branch that merge from + >>> title = "Implement new feature" + >>> body = "This pull request adds functionality X, Y, and Z." + >>> pull_request = await create_pull_request( + base_repo_name=base_repo_name, + head_repo_name=head_repo_name, + base=base, + head=head, + title=title, + body=body, + access_token=get_env("git_access_token") + ) + >>> print(pull_request) + PullRequest(title="Implement new feature", number=26) + + """ + return await GitRepository.create_pull( + base_repo_name=base_repo_name, + head_repo_name=head_repo_name, + base=base, + head=head, + title=title, + body=body, + access_token=access_token, + ) + + +@register_tool(tags=["git"]) +async def create_issue( + access_token: str, + repo_name: str, + title: str, + body: Optional[str] = None, + assignee: Optional[str] = None, + labels: Optional[list[str]] = None, +) -> Issue: + """ + Creates an issue in the specified repository. + + Args: + access_token (str): The access token for authentication. + Visit `https://github.com/settings/tokens` to obtain a personal access token. + For more authentication options, visit: `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html` + repo_name (str): The full repository name (user/repo) where the issue will be created. + title (str): The title of the issue. + body (Optional[str], optional): The body of the issue. Defaults to None. + assignee (Optional[str], optional): The username of the assignee for the issue. Defaults to None. + labels (Optional[list[str]], optional): A list of label names to associate with the issue. Defaults to None. + + + Returns: + Issue: The created issue object. + + Example: + >>> # Create an issue with title and body + >>> repo_name = "username/repository" + >>> title = "Bug Report" + >>> body = "I found a bug in the application." + >>> issue = await create_issue( + repo_name=repo_name, + title=title, + body=body, + access_token=get_env("git_access_token") + ) + >>> print(issue) + Issue(title="Bug Report", number=26) + + >>> # Create an issue with title, body, assignee, and labels + >>> repo_name = "username/repository" + >>> title = "Bug Report" + >>> body = "I found a bug in the application." + >>> assignee = "john_doe" + >>> labels = ["enhancement", "help wanted"] + >>> issue = await create_issue( + repo_name=repo_name, + title=title, + body=body, + assignee=assigee, + labels=labels, + access_token=get_env("git_access_token") + ) + >>> print(issue) + Issue(title="Bug Report", number=26) + """ + return await GitRepository.create_issue( + repo_name=repo_name, title=title, body=body, assignee=assignee, labels=labels, access_token=access_token + ) diff --git a/metagpt/tools/libs/gpt_v_generator.py b/metagpt/tools/libs/gpt_v_generator.py index 4eba3d5ee..baedc3d61 100644 --- a/metagpt/tools/libs/gpt_v_generator.py +++ b/metagpt/tools/libs/gpt_v_generator.py @@ -89,7 +89,7 @@ class GPTvGenerator: webpages_path.mkdir(parents=True, exist_ok=True) index_path = webpages_path / "index.html" - index_path.write_text(CodeParser.parse_code(block=None, text=webpages, lang="html")) + index_path.write_text(CodeParser.parse_code(text=webpages, lang="html")) extract_and_save_code(folder=webpages_path, text=webpages, pattern="styles?.css", language="css") @@ -102,5 +102,5 @@ def extract_and_save_code(folder, text, pattern, language): word = re.search(pattern, text) if word: path = folder / word.group(0) - code = CodeParser.parse_code(block=None, text=text, lang=language) + code = CodeParser.parse_code(text=text, lang=language) path.write_text(code, encoding="utf-8") diff --git a/metagpt/tools/libs/software_development.py b/metagpt/tools/libs/software_development.py index 0149e7e35..f250bcd2d 100644 --- a/metagpt/tools/libs/software_development.py +++ b/metagpt/tools/libs/software_development.py @@ -3,384 +3,10 @@ from __future__ import annotations from pathlib import Path -from typing import Optional -from metagpt.const import ASSISTANT_ALIAS, BUGFIX_FILENAME, REQUIREMENT_FILENAME +from metagpt.const import ASSISTANT_ALIAS from metagpt.logs import ToolLogItem, log_tool_output -from metagpt.schema import BugFixContext, Message from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import any_to_str -from metagpt.utils.project_repo import ProjectRepo - - -@register_tool(tags=["software development", "ProductManager"]) -async def write_prd(idea: str, project_path: Optional[str | Path] = None) -> Path: - """Writes a PRD based on user requirements. - - Args: - idea (str): The idea or concept for the PRD. - project_path (Optional[str|Path], optional): The path to an existing project directory. - If it's None, a new project path will be created. Defaults to None. - - Returns: - Path: The path to the PRD files under the project directory - - Example: - >>> # Create a new project: - >>> from metagpt.tools.libs.software_development import write_prd - >>> prd_path = await write_prd("Create a new feature for the application") - >>> print(prd_path) - '/path/to/project_path/docs/prd/' - - >>> # Add user requirements to the exists project: - >>> from metagpt.tools.libs.software_development import write_prd - >>> project_path = '/path/to/exists_project_path' - >>> prd_path = await write_prd("Create a new feature for the application", project_path=project_path) - >>> print(prd_path = ) - '/path/to/project_path/docs/prd/' - """ - from metagpt.actions import UserRequirement - from metagpt.context import Context - from metagpt.roles import ProductManager - - log_tool_output(output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_prd.__name__)], tool_name=write_prd.__name__) - - ctx = Context() - if project_path and Path(project_path).exists(): - ctx.config.project_path = Path(project_path) - ctx.config.inc = True - - role = ProductManager(context=ctx) - msg = await role.run(with_message=Message(content=idea, cause_by=UserRequirement)) - await role.run(with_message=msg) - - outputs = [ - ToolLogItem(name="Intermedia PRD File", value=str(ctx.repo.docs.prd.workdir / i)) - for i in ctx.repo.docs.prd.changed_files.keys() - ] - outputs.extend( - [ - ToolLogItem(name="PRD File", value=str(ctx.repo.resources.prd.workdir / i)) - for i in ctx.repo.resources.prd.changed_files.keys() - ] - ) - outputs.extend( - [ - ToolLogItem(name="Competitive Analysis", value=str(ctx.repo.resources.competitive_analysis.workdir / i)) - for i in ctx.repo.resources.competitive_analysis.changed_files.keys() - ] - ) - log_tool_output(output=outputs, tool_name=write_prd.__name__) - - return ctx.repo.docs.prd.workdir - - -@register_tool(tags=["Design", "software development", "Architect"]) -async def write_design(prd_path: str | Path) -> Path: - """Writes a system design to the project repository, based on the PRD of the project. - - Args: - prd_path (str|Path): The path to the PRD files under the project directory. - - Returns: - Path: The path to the system design files under the project directory. - - Example: - >>> from metagpt.tools.libs.software_development import write_design - >>> prd_path = '/path/to/project_path/docs/prd' # Returned by `write_prd` - >>> system_design_path = await write_desgin(prd_path) - >>> print(system_design_path) - '/path/to/project_path/docs/system_design/' - - """ - from metagpt.actions import WritePRD - from metagpt.context import Context - from metagpt.roles import Architect - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_design.__name__)], tool_name=write_design.__name__ - ) - - ctx = Context() - prd_path = Path(prd_path) - project_path = (Path(prd_path) if not prd_path.is_file() else prd_path.parent) / "../.." - ctx.set_repo_dir(project_path) - - role = Architect(context=ctx) - await role.run(with_message=Message(content="", cause_by=WritePRD)) - - outputs = [ - ToolLogItem(name="Intermedia Design File", value=str(ctx.repo.docs.system_design.workdir / i)) - for i in ctx.repo.docs.system_design.changed_files.keys() - ] - for i in ctx.repo.resources.system_design.changed_files.keys(): - outputs.append(ToolLogItem(name="Design File", value=str(ctx.repo.resources.system_design.workdir / i))) - for i in ctx.repo.resources.data_api_design.changed_files.keys(): - outputs.append( - ToolLogItem(name="Class Diagram File", value=str(ctx.repo.resources.data_api_design.workdir / i)) - ) - for i in ctx.repo.resources.seq_flow.changed_files.keys(): - outputs.append(ToolLogItem(name="Sequence Diagram File", value=str(ctx.repo.resources.seq_flow.workdir / i))) - log_tool_output(output=outputs, tool_name=write_design.__name__) - - return ctx.repo.docs.system_design.workdir - - -@register_tool(tags=["software development", "Architect"]) -async def write_project_plan(system_design_path: str | Path) -> Path: - """Writes a project plan to the project repository, based on the design of the project. - - Args: - system_design_path (str|Path): The path to the system design files under the project directory. - - Returns: - Path: The path to task files under the project directory. - - Example: - >>> from metagpt.tools.libs.software_development import write_project_plan - >>> system_design_path = '/path/to/project_path/docs/system_design/' # Returned by `write_design` - >>> task_path = await write_project_plan(system_design_path) - >>> print(task_path) - '/path/to/project_path/docs/task' - - """ - from metagpt.actions import WriteDesign - from metagpt.context import Context - from metagpt.roles import ProjectManager - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_project_plan.__name__)], - tool_name=write_project_plan.__name__, - ) - - ctx = Context() - system_design_path = Path(system_design_path) - project_path = (system_design_path if not system_design_path.is_file() else system_design_path.parent) / "../.." - ctx.set_repo_dir(project_path) - - role = ProjectManager(context=ctx) - await role.run(with_message=Message(content="", cause_by=WriteDesign)) - - outputs = [ - ToolLogItem(name="Intermedia Project Plan", value=str(ctx.repo.docs.task.workdir / i)) - for i in ctx.repo.docs.task.changed_files.keys() - ] - outputs.extend( - [ - ToolLogItem(name="Project Plan", value=str(ctx.repo.resources.api_spec_and_task.workdir / i)) - for i in ctx.repo.resources.api_spec_and_task.changed_files.keys() - ] - ) - log_tool_output(output=outputs, tool_name=write_project_plan.__name__) - - return ctx.repo.docs.task.workdir - - -@register_tool(tags=["software development", "Engineer"]) -async def write_codes(task_path: str | Path, inc: bool = False) -> Path: - """Writes code to implement designed features according to the project plan and adds them to the project repository. - In code writing tasks, prioritize calling this tool against writing code from scratch directly. - - Args: - task_path (str|Path): The path to task files under the project directory. - inc (bool, optional): Whether to write incremental codes. Defaults to False. - - Returns: - Path: The path to the source code files under the project directory. - - Example: - # Write codes to a new project - >>> from metagpt.tools.libs.software_development import write_codes - >>> task_path = '/path/to/project_path/docs/task' # Returned by `write_project_plan` - >>> src_path = await write_codes(task_path) - >>> print(src_path) - '/path/to/project_path/src/' - - # Write increment codes to the exists project - >>> from metagpt.tools.libs.software_development import write_codes - >>> task_path = '/path/to/project_path/docs/task' # Returned by `write_prd` - >>> src_path = await write_codes(task_path, inc=True) - >>> print(src_path) - '/path/to/project_path/src/' - """ - from metagpt.actions import WriteTasks - from metagpt.context import Context - from metagpt.roles import Engineer - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_codes.__name__)], tool_name=write_codes.__name__ - ) - - ctx = Context() - ctx.config.inc = inc - task_path = Path(task_path) - project_path = (task_path if not task_path.is_file() else task_path.parent) / "../.." - ctx.set_repo_dir(project_path) - - role = Engineer(context=ctx) - msg = Message(content="", cause_by=WriteTasks, send_to=role) - me = {any_to_str(role), role.name} - while me.intersection(msg.send_to): - msg = await role.run(with_message=msg) - - outputs = [ - ToolLogItem(name="Source File", value=str(ctx.repo.srcs.workdir / i)) - for i in ctx.repo.srcs.changed_files.keys() - ] - log_tool_output(output=outputs, tool_name=write_codes.__name__) - - return ctx.repo.srcs.workdir - - -@register_tool(tags=["software development", "QaEngineer"]) -async def run_qa_test(src_path: str | Path) -> Path: - """Run QA test on the project repository. - - Args: - src_path (str|Path): The path to the source code files under the project directory. - - Returns: - Path: The path to the unit tests under the project directory - - Example: - >>> from metagpt.tools.libs.software_development import run_qa_test - >>> src_path = '/path/to/project_path/src/' # Returned by `write_codes` - >>> test_path = await run_qa_test(src_path) - >>> print(test_path) - '/path/to/project_path/tests' - """ - from metagpt.actions.summarize_code import SummarizeCode - from metagpt.context import Context - from metagpt.environment import Environment - from metagpt.roles import QaEngineer - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=run_qa_test.__name__)], tool_name=run_qa_test.__name__ - ) - - ctx = Context() - src_path = Path(src_path) - project_path = (src_path if not src_path.is_file() else src_path.parent) / ".." - ctx.set_repo_dir(project_path) - ctx.src_workspace = ctx.git_repo.workdir / ctx.git_repo.workdir.name - - env = Environment(context=ctx) - role = QaEngineer(context=ctx) - env.add_role(role) - - msg = Message(content="", cause_by=SummarizeCode, send_to=role) - env.publish_message(msg) - - while not env.is_idle: - await env.run() - - outputs = [ - ToolLogItem(name="Unit Test File", value=str(ctx.repo.tests.workdir / i)) - for i in ctx.repo.tests.changed_files.keys() - ] - log_tool_output(output=outputs, tool_name=run_qa_test.__name__) - - return ctx.repo.tests.workdir - - -@register_tool(tags=["software development", "Engineer"]) -async def fix_bug(project_path: str | Path, issue: str) -> Path: - """Fix bugs in the project repository. - - Args: - project_path (str|Path): The path to the project repository. - issue (str): Description of the bug or issue. - - Returns: - Path: The path to the project directory - - Example: - >>> from metagpt.tools.libs.software_development import fix_bug - >>> project_path = '/path/to/project_path' # Returned by `write_codes` - >>> issue = 'Exception: exception about ...; Bug: bug about ...; Issue: issue about ...' - >>> project_path = await fix_bug(project_path=project_path, issue=issue) - >>> print(project_path) - '/path/to/project_path' - """ - from metagpt.actions.fix_bug import FixBug - from metagpt.context import Context - from metagpt.roles import Engineer - - log_tool_output(output=[ToolLogItem(name=ASSISTANT_ALIAS, value=fix_bug.__name__)], tool_name=fix_bug.__name__) - - ctx = Context() - ctx.set_repo_dir(project_path) - ctx.src_workspace = ctx.git_repo.workdir / ctx.git_repo.workdir.name - await ctx.repo.docs.save(filename=BUGFIX_FILENAME, content=issue) - await ctx.repo.docs.save(filename=REQUIREMENT_FILENAME, content="") - - role = Engineer(context=ctx) - bug_fix = BugFixContext(filename=BUGFIX_FILENAME) - msg = Message( - content=bug_fix.model_dump_json(), - instruct_content=bug_fix, - role="", - cause_by=FixBug, - sent_from=role, - send_to=role, - ) - me = {any_to_str(role), role.name} - while me.intersection(msg.send_to): - msg = await role.run(with_message=msg) - - outputs = [ - ToolLogItem(name="Changed File", value=str(ctx.repo.srcs.workdir / i)) - for i in ctx.repo.srcs.changed_files.keys() - ] - log_tool_output(output=outputs, tool_name=fix_bug.__name__) - - return project_path - - -@register_tool(tags=["software development", "git"]) -async def git_archive(project_path: str | Path) -> str: - """Stage and commit changes for the project repository using Git. - - Args: - project_path (str|Path): The path to the project repository. - - - Returns: - git log - - Example: - >>> from metagpt.tools.libs.software_development import git_archive - >>> project_path = '/path/to/project_path' # Returned by `write_prd` - >>> git_log = await git_archive(project_path=project_path) - >>> print(git_log) - commit a221d1c418c07f2b4fc07001e486285ead1a520a (HEAD -> feature/toollib/software_company, geekan/main) - Merge: e01afd09 4a72f398 - Author: Sirui Hong - Date: Tue Mar 19 15:16:03 2024 +0800 - Merge pull request #1037 from iorisa/fixbug/issues/1018 - fixbug: #1018 - - """ - from metagpt.context import Context - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=git_archive.__name__)], tool_name=git_archive.__name__ - ) - - ctx = Context() - project_dir = ProjectRepo.search_project_path(project_path) - if not project_dir: - ValueError(f"{project_path} is not a valid git repository.") - ctx.set_repo_dir(project_dir) - files = " ".join(ctx.git_repo.changed_files.keys()) - outputs = [ToolLogItem(name="cmd", value=f"git add {files}")] - log_tool_output(output=outputs, tool_name=git_archive.__name__) - ctx.git_repo.archive() - - outputs = [ToolLogItem(name="cmd", value="git commit -m 'Archive'")] - log_tool_output(output=outputs, tool_name=git_archive.__name__) - - return ctx.git_repo.log() @register_tool(tags=["software development", "import git repo"]) diff --git a/metagpt/tools/tool_recommend.py b/metagpt/tools/tool_recommend.py index 01ff61834..05e8e1400 100644 --- a/metagpt/tools/tool_recommend.py +++ b/metagpt/tools/tool_recommend.py @@ -132,7 +132,7 @@ class ToolRecommender(BaseModel): topk=topk, ) rsp = await LLM().aask(prompt, stream=False) - rsp = CodeParser.parse_code(block=None, text=rsp) + rsp = CodeParser.parse_code(text=rsp) ranked_tools = json.loads(rsp) valid_tools = validate_tool_names(ranked_tools) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 9c3436172..e2520ef13 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -271,7 +271,7 @@ class CodeParser: return block_dict @classmethod - def parse_code(cls, block: Optional[str], text: str, lang: str = "") -> str: + def parse_code(cls, text: str, lang: str = "", block: Optional[str] = None) -> str: if block: text = cls.parse_block(block, text) pattern = rf"```{lang}.*?\s+(.*?)```" @@ -287,7 +287,7 @@ class CodeParser: @classmethod def parse_str(cls, block: str, text: str, lang: str = ""): - code = cls.parse_code(block, text, lang) + code = cls.parse_code(block=block, text=text, lang=lang) code = code.split("=")[-1] code = code.strip().strip("'").strip('"') return code @@ -295,7 +295,7 @@ class CodeParser: @classmethod def parse_file_list(cls, block: str, text: str, lang: str = "") -> list[str]: # Regular expression pattern to find the tasks list. - code = cls.parse_code(block, text, lang) + code = cls.parse_code(block=block, text=text, lang=lang) # print(code) pattern = r"\s*(.*=.*)?(\[.*\])" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 2d2927806..4ca166b73 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,14 +8,22 @@ """ from __future__ import annotations +import re import shutil import uuid from enum import Enum from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Optional from git.repo import Repo from git.repo.fun import is_git_dir +from github import Auth, Github +from github.GithubObject import NotSet +from github.Issue import Issue +from github.Label import Label +from github.Milestone import Milestone +from github.NamedUser import NamedUser +from github.PullRequest import PullRequest from gitignore_parser import parse_gitignore from tenacity import retry, stop_after_attempt, wait_random_exponential @@ -35,6 +43,12 @@ class ChangeType(Enum): UNTRACTED = "U" # File is untracked (not added to version control) +class RateLimitError(Exception): + def __init__(self, message="Rate limit exceeded"): + self.message = message + super().__init__(self.message) + + class GitRepository: """A class representing a Git repository. @@ -322,3 +336,156 @@ class GitRepository: def log(self) -> str: """Return git log""" return self._repository.git.log() + + @staticmethod + async def create_pull( + base: str, + head: str, + base_repo_name: str, + head_repo_name: Optional[str] = None, + *, + title: Optional[str] = None, + body: Optional[str] = None, + maintainer_can_modify: Optional[bool] = None, + draft: Optional[bool] = None, + issue: Optional[Issue] = None, + access_token: Optional[str] = None, + auth: Optional[Auth] = None, + ) -> PullRequest: + """ + Creates a pull request in the specified repository. + + Args: + base (str): The name of the base branch. + head (str): The name of the head branch. + base_repo_name (str): The full repository name (user/repo) where the pull request will be created. + head_repo_name (Optional[str], optional): The full repository name (user/repo) where the pull request will merge from. Defaults to None. + title (Optional[str], optional): The title of the pull request. Defaults to None. + body (Optional[str], optional): The body of the pull request. Defaults to None. + maintainer_can_modify (Optional[bool], optional): Whether maintainers can modify the pull request. Defaults to None. + draft (Optional[bool], optional): Whether the pull request is a draft. Defaults to None. + issue (Optional[Issue], optional): The issue linked to the pull request. Defaults to None. + access_token (Optional[str], optional): The access token for authentication. Defaults to None. Visit `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html`, `https://github.com/PyGithub/PyGithub/blob/main/doc/examples/Authentication.rst`. + auth (Optional[Auth], optional): The authentication method. Defaults to None. Visit `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html` + + Returns: + PullRequest: The created pull request object. + """ + title = title or NotSet + body = body or NotSet + maintainer_can_modify = maintainer_can_modify or NotSet + draft = draft or NotSet + issue = issue or NotSet + if not auth and not access_token: + raise ValueError('`access_token` is invalid. Visit: "https://github.com/settings/tokens"') + auth = auth or Auth.Token(access_token) + g = Github(auth=auth) + base_repo = g.get_repo(base_repo_name) + head_repo = g.get_repo(head_repo_name) if head_repo_name and head_repo_name != base_repo_name else None + x_ratelimit_remaining = base_repo.raw_headers.get("x-ratelimit-remaining") + if ( + x_ratelimit_remaining + and bool(re.match(r"^-?\d+$", x_ratelimit_remaining)) + and int(x_ratelimit_remaining) <= 0 + ): + raise RateLimitError() + if not head_repo: + pr = base_repo.create_pull( + base=base, + head=head, + title=title, + body=body, + maintainer_can_modify=maintainer_can_modify, + draft=draft, + issue=issue, + ) + else: + head_branch = base_repo.get_branch(base) + base_branch = head_repo.get_branch(head) + pr = base_repo.create_pull( + base=base_branch.name, + head=head_branch.commit.sha, + title=title, + body=body, + maintainer_can_modify=maintainer_can_modify, + draft=draft, + issue=issue, + ) + return pr + + @staticmethod + async def create_issue( + repo_name: str, + title: str, + body: Optional[str] = None, + assignee: NamedUser | Optional[str] = None, + milestone: Optional[Milestone] = None, + labels: list[Label] | Optional[list[str]] = None, + assignees: Optional[list[str]] | list[NamedUser] = None, + access_token: Optional[str] = None, + auth: Optional[Auth] = None, + ) -> Issue: + """ + Creates an issue in the specified repository. + + Args: + repo_name (str): The full repository name (user/repo) where the issue will be created. + title (str): The title of the issue. + body (Optional[str], optional): The body of the issue. Defaults to None. + assignee (Union[NamedUser, str], optional): The assignee for the issue, either as a NamedUser object or their username. Defaults to None. + milestone (Optional[Milestone], optional): The milestone to associate with the issue. Defaults to None. + labels (Union[list[Label], list[str]], optional): The labels to associate with the issue, either as Label objects or their names. Defaults to None. + assignees (Union[list[str], list[NamedUser]], optional): The list of usernames or NamedUser objects to assign to the issue. Defaults to None. + access_token (Optional[str], optional): The access token for authentication. Defaults to None. Visit `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html`, `https://github.com/PyGithub/PyGithub/blob/main/doc/examples/Authentication.rst`. + auth (Optional[Auth], optional): The authentication method. Defaults to None. Visit `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html` + + Returns: + Issue: The created issue object. + """ + body = body or NotSet + assignee = assignee or NotSet + milestone = milestone or NotSet + labels = labels or NotSet + assignees = assignees or NotSet + if not auth and not access_token: + raise ValueError('`access_token` is invalid. Visit: "https://github.com/settings/tokens"') + auth = auth or Auth.Token(access_token) + g = Github(auth=auth) + + repo = g.get_repo(repo_name) + x_ratelimit_remaining = repo.raw_headers.get("x-ratelimit-remaining") + if ( + x_ratelimit_remaining + and bool(re.match(r"^-?\d+$", x_ratelimit_remaining)) + and int(x_ratelimit_remaining) <= 0 + ): + raise RateLimitError() + issue = repo.create_issue( + title=title, + body=body, + assignee=assignee, + milestone=milestone, + labels=labels, + assignees=assignees, + ) + return issue + + @staticmethod + async def get_repos(access_token: Optional[str] = None, auth: Optional[Auth] = None) -> List[str]: + """ + Fetches a list of public repositories belonging to the authenticated user. + + Args: + access_token (Optional[str], optional): The access token for authentication. Defaults to None. + Visit `https://github.com/settings/tokens` for obtaining a personal access token. + auth (Optional[Auth], optional): The authentication method. Defaults to None. + Visit `https://pygithub.readthedocs.io/en/latest/examples/Authentication.html` for more information. + + Returns: + List[str]: A list of full names of the public repositories belonging to the user. + """ + auth = auth or Auth.Token(access_token) + git = Github(auth=auth) + user = git.get_user() + v = user.get_repos(visibility="public") + return [i.full_name for i in v] diff --git a/requirements.txt b/requirements.txt index 2cf393502..b40c69c9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -70,4 +70,5 @@ qianfan==0.3.2 dashscope==1.14.1 rank-bm25==0.2.2 # for tool recommendation gymnasium==0.29.1 -pylint~=3.0.3 \ No newline at end of file +pylint~=3.0.3 +pygithub~=2.3 \ No newline at end of file diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 1709e1f5b..42623f807 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -110,7 +110,7 @@ async def test_write_refined_code(context, git_dir): # old_workspace contains the legacy code await context.repo.with_src_path(context.repo.old_workspace).srcs.save( - filename="game.py", content=CodeParser.parse_code(block="", text=REFINED_CODE_INPUT_SAMPLE) + filename="game.py", content=CodeParser.parse_code(text=REFINED_CODE_INPUT_SAMPLE) ) ccontext = CodingContext( diff --git a/tests/metagpt/actions/test_write_code_plan_and_change_an.py b/tests/metagpt/actions/test_write_code_plan_and_change_an.py index 5c262b4b7..5bc860469 100644 --- a/tests/metagpt/actions/test_write_code_plan_and_change_an.py +++ b/tests/metagpt/actions/test_write_code_plan_and_change_an.py @@ -45,7 +45,7 @@ async def test_write_code_plan_and_change_an(mocker, context, git_dir): await context.repo.docs.task.save(filename="2.json", content=json.dumps(REFINED_TASK_JSON)) await context.repo.with_src_path(context.repo.old_workspace).srcs.save( - filename="game.py", content=CodeParser.parse_code(block="", text=REFINED_CODE_INPUT_SAMPLE) + filename="game.py", content=CodeParser.parse_code(text=REFINED_CODE_INPUT_SAMPLE) ) root = ActionNode.from_children( diff --git a/tests/metagpt/environment/mgx_env/run_mgx_env.py b/tests/metagpt/environment/mgx_env/run_mgx_env.py new file mode 100644 index 000000000..86678c97c --- /dev/null +++ b/tests/metagpt/environment/mgx_env/run_mgx_env.py @@ -0,0 +1,78 @@ +import asyncio +import threading + +from metagpt.environment.mgx.mgx_env import MGXEnv +from metagpt.roles import ( + Architect, + Engineer, + ProductManager, + ProjectManager, + QaEngineer, +) +from metagpt.roles.di.data_analyst import DataAnalyst +from metagpt.roles.di.team_leader import TeamLeader +from metagpt.schema import Message + + +async def main(requirement, enable_human_input=False): + env = MGXEnv() + env.add_roles( + [ + TeamLeader(), + ProductManager(), + Architect(), + ProjectManager(), + Engineer(n_borg=5, use_code_review=False), + QaEngineer(), + DataAnalyst(tools=[""]), + ] + ) + + if enable_human_input: + # simulate human sending messages in chatbox + send_human_input(env) + + env.publish_message(Message(content=requirement)) + + while not env.is_idle: + await env.run() + + +def send_human_input(env): + """ + Simulate sending message in chatbox + Note in local environment, the message is consumed only after current round of env.run is finished + """ + + def send_messages(): + while True: + message = input("Enter a message any time: ") + env.publish_message(Message(content=message)) + + # Start a thread for sending messages + send_thread = threading.Thread(target=send_messages, args=()) + send_thread.start() + + +GAME_REQ = "create a 2048 game" +SIMPLE_REQ = "print statistic summary of sklearn iris dataset" +WINE_REQ = "Run data analysis on sklearn Wine recognition dataset, include a plot, and train a model to predict wine class (20% as validation), and show validation accuracy." +PAPER_LIST_REQ = """ +Get data from `paperlist` table in https://papercopilot.com/statistics/iclr-statistics/iclr-2024-statistics/, +and save it to a csv file. paper title must include `multiagent` or `large language model`. *notice: print key variables* +""" +ECOMMERCE_REQ = """ +Get products data from website https://scrapeme.live/shop/ and save it as a csv file. +**Notice: Firstly parse the web page encoding and the text HTML structure; +The first page product name, price, product URL, and image URL must be saved in the csv;** +""" +data_path = "data/titanic" +train_path = f"{data_path}/split_train.csv" +eval_path = f"{data_path}/split_eval.csv" +TITANIC_REQ = f"This is a titanic passenger survival dataset, your goal is to predict passenger survival outcome. The target column is Survived. Perform data analysis, data preprocessing, feature engineering, and modeling to predict the target. Report accuracy on the eval data. Train data path: '{train_path}', eval data path: '{eval_path}'." + + +if __name__ == "__main__": + # NOTE: Change the requirement to the one you want to test + # Set enable_human_input to True if you want to simulate sending messages in chatbox + asyncio.run(main(requirement=SIMPLE_REQ, enable_human_input=False)) diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index d263a8a2f..d5eae662f 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -91,7 +91,7 @@ target_code = """task_list = [ def test_parse_code(): - code = CodeParser.parse_code("Task list", TASKS, lang="python") + code = CodeParser.parse_code(block="Task list", text=TASKS, lang="python") logger.info(code) assert isinstance(code, str) assert target_code == code diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 7559655d3..07f5b4305 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -11,17 +11,44 @@ from pathlib import Path import pytest from metagpt.actions import UserRequirement +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.context import Context from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.roles import Architect, ProductManager, Role -from metagpt.schema import Message +from metagpt.roles import ( + Architect, + Engineer, + ProductManager, + ProjectManager, + QaEngineer, + Role, +) +from metagpt.schema import Message, UserMessage +from metagpt.utils.common import any_to_str, is_send_to serdeser_path = Path(__file__).absolute().parent.joinpath("../data/serdeser_storage") +class MockEnv(Environment): + def publish_message(self, message: Message, peekable: bool = True) -> bool: + consumers = [] + for role, addrs in self.member_addrs.items(): + if is_send_to(message, addrs): + role.put_message(message) + consumers.append(role) + if not consumers: + logger.warning(f"Message no recipients: {message.dump()}") + if message.cause_by in [any_to_str(UserRequirement), any_to_str(PrepareDocuments)]: + assert len(consumers) == 1 + + return True + + @pytest.fixture def env(): - return Environment() + context = Context() + context.kwargs.tag = __file__ + return MockEnv(context=context) def test_add_role(env: Environment): @@ -54,10 +81,56 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) - env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) + env.publish_message(UserMessage(content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement, send_to=product_manager)) await env.run(k=2) - logger.info(f"{env.history=}") - assert len(env.history) > 10 + logger.info(f"{env.history}") + assert len(env.history.storage) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("content", "send_to"), + [ + ("snake game", any_to_str(ProductManager)), + ( + "Rewrite the PRD file of the project at '/Users/iorishinier/github/MetaGPT/workspace/snake_game', add 'moving enemy' to the original requirement", + any_to_str(ProductManager), + ), + ( + "Add 'random moving enemy, and dispears after 10 seconds' design to the project at '/Users/iorishinier/github/MetaGPT/workspace/snake_game'", + any_to_str(Architect), + ), + ( + 'Rewrite the tasks file of the project at "/Users/iorishinier/github/MetaGPT/workspace/snake_game"', + any_to_str(ProjectManager), + ), + ( + "Rewrite 'main.py' of the project at '/Users/iorishinier/github/MetaGPT/workspace/snake_game'", + any_to_str(Engineer), + ), + ( + "Rewrite the unit test of 'main.py' at '/Users/iorishinier/github/MetaGPT/workspace/snake_game'", + any_to_str(QaEngineer), + ), + ], +) +async def test_env(content, send_to): + context = Context() + env = MockEnv(context=context) + env.add_roles( + [ + ProductManager(context=context), + Architect(context=context), + ProjectManager(context=context), + Engineer(n_borg=5, use_code_review=True, context=context), + QaEngineer(context=context, test_round_allowed=2), + ] + ) + msg = UserMessage(content=content, send_to=send_to) + env.publish_message(msg) + while not env.is_idle: + await env.run() + pass if __name__ == "__main__": diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 22f6ae9fb..6f54b062d 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -350,5 +350,47 @@ class TestPlan: assert plan.current_task_id == "2" +@pytest.mark.parametrize( + ("content", "key_descriptions"), + [ + ( + """ +Traceback (most recent call last): + File "/Users/iorishinier/github/MetaGPT/workspace/game_2048_1/game_2048/main.py", line 38, in + Main().main() + File "/Users/iorishinier/github/MetaGPT/workspace/game_2048_1/game_2048/main.py", line 28, in main + self.user_interface.draw() + File "/Users/iorishinier/github/MetaGPT/workspace/game_2048_1/game_2048/user_interface.py", line 16, in draw + if grid[i][j] != 0: +TypeError: 'Grid' object is not subscriptable + """, + { + "filename": "the string type of the path name of the source code where the bug resides", + "line": "the integer type of the line error occurs", + "function_name": "the string type of the function name the error occurs in", + "code": "the string type of the codes where the error occurs at", + "info": "the string type of the error information", + }, + ), + ( + "将代码提交到github上的iorisa/repo1的branch1分支，发起pull request ，合并到master分支。", + { + "repo_name": "the string type of github repo to create pull", + "head": "the string type of github branch to be pushed", + "base": "the string type of github branch to merge the changes into", + }, + ), + ], +) +async def test_parse_resources(context, content: str, key_descriptions): + msg = Message(content=content) + llm = context.llm_with_cost_manager_from_llm_config(context.config.llm) + result = await msg.parse_resources(llm=llm, key_descriptions=key_descriptions) + assert result + assert result.get("resources") + for k in key_descriptions.keys(): + assert k in result + + if __name__ == "__main__": pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/tools/libs/test_git.py b/tests/metagpt/tools/libs/test_git.py index 12192ca86..a20a0c545 100644 --- a/tests/metagpt/tools/libs/test_git.py +++ b/tests/metagpt/tools/libs/test_git.py @@ -1,7 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +from __future__ import annotations + +import os import pytest +from github import Auth, Github from pydantic import BaseModel from metagpt.tools.libs.git import git_checkout, git_clone @@ -13,10 +17,15 @@ class SWEBenchItem(BaseModel): repo: str +def get_env(key): + return os.environ.get(key) + + @pytest.mark.asyncio @pytest.mark.parametrize( ["url", "commit_id"], [("https://github.com/sqlfluff/sqlfluff.git", "d19de0ecd16d298f9e3bfb91da122734c40c01e5")] ) +@pytest.mark.skip async def test_git(url: str, commit_id: str): repo_dir = await git_clone(url) assert repo_dir @@ -27,5 +36,67 @@ async def test_git(url: str, commit_id: str): repo.delete_repository() +@pytest.mark.skip +def test_login(): + auth = Auth.Login(get_env("GITHUB_USER"), get_env("GITHUB_PWD")) + g = Github(auth=auth) + repo = g.get_repo("geekan/MetaGPT") + topics = repo.get_topics() + assert topics + open_issues = repo.get_issues(state="open") + issues = [i for i in open_issues] + assert issues + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_new_issue(): + issue = await GitRepository.create_issue( + repo_name="iorisa/MetaGPT", + title="This is a new issue", + body="This is the issue body", + access_token=get_env("GITHUB_PERSONAL_ACCESS_TOKEN"), + ) + print(issue) + assert issue.number + pass + + +@pytest.mark.skip +@pytest.mark.asyncio +async def test_new_pr(): + body = """ + >>> SUMMARY + >>> Change HTTP library used to send requests + >>> + >>> TESTS + >>> - [x] Send 'GET' request + >>> - [x] Send 'POST' request with/without body + """ + pr = await GitRepository.create_pull( + repo_name="iorisa/MetaGPT", + base="send18", + head="fixbug/gbk", + title="Test pr", + body=body, + access_token=get_env("GITHUB_PERSONAL_ACCESS_TOKEN"), + ) + print(pr) + assert pr + + +@pytest.mark.skip +def test_auth(): + access_token = get_env("GITHUB_PERSONAL_ACCESS_TOKEN") + auth = Auth.Token(access_token) + g = Github(auth=auth) + u = g.get_user() + v = u.get_repos(visibility="public") + a = [i.full_name for i in v] + assert a + print(a) + pass + + if __name__ == "__main__": pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_code_parser.py b/tests/metagpt/utils/test_code_parser.py index 294324b8f..f4d822f85 100644 --- a/tests/metagpt/utils/test_code_parser.py +++ b/tests/metagpt/utils/test_code_parser.py @@ -119,7 +119,7 @@ class TestCodeParser: assert "game.py" in result def test_parse_code(self, parser, text): - result = parser.parse_code("Task list", text, "python") + result = parser.parse_code(block="Task list", text=text, lang="python") print(result) assert "game.py" in result diff --git a/tests/mock/mock_llm.py b/tests/mock/mock_llm.py index f6c206d5e..168125448 100644 --- a/tests/mock/mock_llm.py +++ b/tests/mock/mock_llm.py @@ -85,7 +85,7 @@ class MockLLM(OriginalLLM): format_msgs: Optional[list[dict[str, str]]] = None, images: Optional[Union[str, list[str]]] = None, timeout=LLM_API_TIMEOUT, - stream=True, + stream=False, ) -> str: # used to identify it a message has been called before if isinstance(msg, list):