diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 5ed31bed8..b027616f7 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -70,6 +70,6 @@ class DebugError(Action): prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=rsp) + code = CodeParser.parse_code(text=rsp) return code diff --git a/metagpt/actions/di/write_analysis_code.py b/metagpt/actions/di/write_analysis_code.py index 711e56d39..548555196 100644 --- a/metagpt/actions/di/write_analysis_code.py +++ b/metagpt/actions/di/write_analysis_code.py @@ -30,7 +30,7 @@ class WriteAnalysisCode(Action): ) rsp = await self._aask(reflection_prompt, system_msgs=[REFLECTION_SYSTEM_MSG]) - reflection = json.loads(CodeParser.parse_code(block=None, text=rsp)) + reflection = json.loads(CodeParser.parse_code(text=rsp)) return reflection["improved_impl"] @@ -57,7 +57,7 @@ class WriteAnalysisCode(Action): code = await self._debug_with_reflection(context=context, working_memory=working_memory) else: rsp = await self.llm.aask(context, system_msgs=[INTERPRETER_SYSTEM_MSG], **kwargs) - code = CodeParser.parse_code(block=None, text=rsp) + code = CodeParser.parse_code(text=rsp) return code @@ -69,5 +69,5 @@ class CheckData(Action): code_written = "\n\n".join(code_written) prompt = CHECK_DATA_PROMPT.format(code_written=code_written) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block=None, text=rsp) + code = CodeParser.parse_code(text=rsp) return code diff --git a/metagpt/actions/di/write_plan.py b/metagpt/actions/di/write_plan.py index 8d6eccf57..efea9f526 100644 --- a/metagpt/actions/di/write_plan.py +++ b/metagpt/actions/di/write_plan.py @@ -47,7 +47,7 @@ class WritePlan(Action): context="\n".join([str(ct) for ct in context]), max_tasks=max_tasks, task_type_desc=task_type_desc ) rsp = await self._aask(prompt) - rsp = CodeParser.parse_code(block=None, text=rsp) + rsp = CodeParser.parse_code(text=rsp) return rsp diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 08f2c2fcb..bba3deaa3 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -9,10 +9,12 @@ """ import shutil from pathlib import Path -from typing import Optional +from typing import Dict, Optional -from metagpt.actions import Action, ActionOutput +from metagpt.actions import Action, UserRequirement from metagpt.const import REQUIREMENT_FILENAME +from metagpt.schema import AIMessage +from metagpt.utils.common import any_to_str from metagpt.utils.file_repository import FileRepository @@ -21,6 +23,13 @@ class PrepareDocuments(Action): name: str = "PrepareDocuments" i_context: Optional[str] = None + key_descriptions: Optional[Dict[str, str]] = None + send_to: str + + def __init__(self, **kwargs): + super().__init__(**kwargs) + if not self.key_descriptions: + self.key_descriptions = {"project_path": 'the project path if exists in "Original Requirement"'} @property def config(self): @@ -40,10 +49,26 @@ class PrepareDocuments(Action): async def run(self, with_messages, **kwargs): """Create and initialize the workspace folder, initialize the Git environment.""" + user_requirements = [i for i in with_messages if i.cause_by == any_to_str(UserRequirement)] + if not self.config.project_path and user_requirements and self.key_descriptions: + args = await user_requirements[0].parse_resources(llm=self.llm, key_descriptions=self.key_descriptions) + for k, v in args.items(): + if not v or k in ["resources", "reason"]: + continue + self.context.kwargs.set(k, v) + if self.context.kwargs.project_path: + self.config.update_via_cli( + project_path=self.context.kwargs.project_path, + project_name="", + inc=False, + reqa_file=self.context.kwargs.reqa_file or "", + max_auto_summarize_code=0, + ) + self._init_repo() # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content) # Send a Message notification to the WritePRD action, instructing it to process requirements using # `docs/requirement.txt` and `docs/prd/`. - return ActionOutput(content=doc.content, instruct_content=doc) + return AIMessage(content="", instruct_content=doc, cause_by=self, send_to=self.send_to) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index d2fa15f6b..aac3a6d87 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -87,7 +87,7 @@ class WriteCode(Action): @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=code_rsp) + code = CodeParser.parse_code(text=code_rsp) return code async def run(self, *args, **kwargs) -> CodingContext: diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index ac6fe7045..ed868c867 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -132,7 +132,7 @@ class WriteCodeReview(Action): # if LBTM, rewrite code rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}" code_rsp = await self._aask(rewrite_prompt) - code = CodeParser.parse_code(block="", text=code_rsp) + code = CodeParser.parse_code(text=code_rsp) return result, code async def run(self, *args, **kwargs) -> CodingContext: diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 978fa20a6..286d3ea13 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -45,7 +45,7 @@ class WriteTest(Action): code_rsp = await self._aask(prompt) try: - code = CodeParser.parse_code(block="", text=code_rsp) + code = CodeParser.parse_code(text=code_rsp) except Exception: # Handle the exception if needed logger.error(f"Can't parse the code: {code_rsp}") diff --git a/metagpt/const.py b/metagpt/const.py index c01f92adc..67636fad3 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -139,3 +139,6 @@ LLM_API_TIMEOUT = 300 # Assistant alias ASSISTANT_ALIAS = "response" + +# Metadata defines +AGENT = "agent" diff --git a/metagpt/ext/stanford_town/roles/st_role.py b/metagpt/ext/stanford_town/roles/st_role.py index 79f58b07d..4856548f0 100644 --- a/metagpt/ext/stanford_town/roles/st_role.py +++ b/metagpt/ext/stanford_town/roles/st_role.py @@ -181,13 +181,13 @@ class STRole(Role): logger.info(f"Role: {self.name} saved role's memory into {str(self.role_storage_path)}") - async def _observe(self, ignore_memory=False) -> int: + async def _observe(self) -> int: if not self.rc.env: return 0 news = [] if not news: news = self.rc.msg_buffer.pop_all() - old_messages = [] if ignore_memory else self.rc.memory.get() + old_messages = [] if not self.enable_memory else self.rc.memory.get() # Filter out messages of interest. self.rc.news = [ n for n in news if (n.cause_by in self.rc.watch or self.name in n.send_to) and n not in old_messages diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index dbfed72df..120c1d3cb 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -220,7 +220,7 @@ class OpenAILLM(BaseLLM): # The response content is `code``, but it appears in the content instead of the arguments. code_formats = "```" if message.content.startswith(code_formats) and message.content.endswith(code_formats): - code = CodeParser.parse_code(None, message.content) + code = CodeParser.parse_code(text=message.content) return {"language": "python", "code": code} # reponse is message return {"language": "markdown", "code": self.get_choice_text(rsp)} diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 166f8cfd0..465beff05 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -6,9 +6,11 @@ @File : architect.py """ -from metagpt.actions import WritePRD +from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.design_api import WriteDesign +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles.role import Role +from metagpt.utils.common import any_to_str class Architect(Role): @@ -32,8 +34,24 @@ class Architect(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + self.enable_memory = False # Initialize actions specific to the Architect role - self.set_actions([WriteDesign]) + self.set_actions([PrepareDocuments(send_to=any_to_str(self), context=self.context), WriteDesign]) # Set events or actions the Architect should watch or be aware of - self._watch({WritePRD}) + self._watch({UserRequirement, PrepareDocuments, WritePRD}) + + async def _think(self) -> bool: + """Decide what to do""" + mappings = { + any_to_str(UserRequirement): 0, + any_to_str(PrepareDocuments): 1, + any_to_str(WritePRD): 1, + } + for i in self.rc.news: + idx = mappings.get(i.cause_by, -1) + if idx < 0: + continue + self.rc.todo = self.actions[idx] + return bool(self.rc.todo) + return False diff --git a/metagpt/roles/di/data_interpreter.py b/metagpt/roles/di/data_interpreter.py index f574943cc..e147cbbe3 100644 --- a/metagpt/roles/di/data_interpreter.py +++ b/metagpt/roles/di/data_interpreter.py @@ -74,7 +74,7 @@ class DataInterpreter(Role): prompt = REACT_THINK_PROMPT.format(user_requirement=self.user_requirement, context=context) rsp = await self.llm.aask(prompt) - rsp_dict = json.loads(CodeParser.parse_code(block=None, text=rsp)) + rsp_dict = json.loads(CodeParser.parse_code(text=rsp)) self.working_memory.add(Message(content=rsp_dict["thoughts"], role="assistant")) need_action = rsp_dict["state"] self._set_state(0) if need_action else self._set_state(-1) diff --git a/metagpt/roles/di/team_leader.py b/metagpt/roles/di/team_leader.py index bf1619308..1bffc90bc 100644 --- a/metagpt/roles/di/team_leader.py +++ b/metagpt/roles/di/team_leader.py @@ -73,7 +73,7 @@ class TeamLeader(Role): context = self.llm.format_msg(self.get_memories(k=10) + [Message(content=prompt, role="user")]) rsp = await self.llm.aask(context, system_msgs=[SYSTEM_PROMPT]) - self.commands = json.loads(CodeParser.parse_code(block=None, text=rsp)) + self.commands = json.loads(CodeParser.parse_code(text=rsp)) self.rc.memory.add(Message(content=rsp, role="assistant")) return True diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 30fb6f187..e7b5cf219 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -24,8 +24,15 @@ from collections import defaultdict from pathlib import Path from typing import List, Optional, Set -from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.actions import ( + Action, + UserRequirement, + WriteCode, + WriteCodeReview, + WriteTasks, +) from metagpt.actions.fix_bug import FixBug +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST from metagpt.actions.summarize_code import SummarizeCode from metagpt.actions.write_code_plan_and_change_an import WriteCodePlanAndChange @@ -39,6 +46,7 @@ from metagpt.const import ( from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( + AIMessage, CodePlanAndChangeContext, CodeSummarizeContext, CodingContext, @@ -53,6 +61,7 @@ from metagpt.utils.common import ( get_project_srcs_path, init_python_folder, ) +from metagpt.utils.git_repository import ChangeType IS_PASS_PROMPT = """ {context} @@ -93,9 +102,20 @@ class Engineer(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - + self.enable_memory = False self.set_actions([WriteCode]) - self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug, WriteCodePlanAndChange]) + self._watch( + [ + UserRequirement, + PrepareDocuments, + WriteTasks, + SummarizeCode, + WriteCode, + WriteCodeReview, + FixBug, + WriteCodePlanAndChange, + ] + ) self.code_todos = [] self.summarize_todos = [] self.next_todo_action = any_to_name(WriteCode) @@ -130,13 +150,11 @@ class Engineer(Role): dependencies=list(dependencies), content=coding_context.code_doc.content, ) - msg = Message( + AIMessage( content=coding_context.model_dump_json(), instruct_content=coding_context, - role=self.profile, cause_by=WriteCode, ) - self.rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) if not changed_files: @@ -156,13 +174,12 @@ class Engineer(Role): if isinstance(self.rc.todo, SummarizeCode): self.next_todo_action = any_to_name(WriteCode) return await self._act_summarize() - return None + return await self.rc.todo.run(self.rc.history) async def _act_write_code(self): changed_files = await self._act_sp_with_cr(review=self.use_code_review) - return Message( + return AIMessage( content="\n".join(changed_files), - role=self.profile, cause_by=WriteCodeReview if self.use_code_review else WriteCode, send_to=self, sent_from=self, @@ -195,9 +212,8 @@ class Engineer(Role): logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}") if not tasks or self.config.max_auto_summarize_code == 0: - return Message( + return AIMessage( content="", - role=self.profile, cause_by=SummarizeCode, sent_from=self, send_to="Edward", # The name of QaEngineer @@ -205,9 +221,7 @@ class Engineer(Role): # The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. # This parameter is used for debugging the workflow. self.n_summarize += 1 if self.config.max_auto_summarize_code > self.n_summarize else 0 - return Message( - content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self - ) + return AIMessage(content=json.dumps(tasks), cause_by=SummarizeCode, send_to=self, sent_from=self) async def _act_code_plan_and_change(self): """Write code plan and change that guides subsequent WriteCode and WriteCodeReview""" @@ -229,9 +243,8 @@ class Engineer(Role): dependencies=dependencies, ) - return Message( + return AIMessage( content=code_plan_and_change, - role=self.profile, cause_by=WriteCodePlanAndChange, send_to=self, sent_from=self, @@ -245,14 +258,25 @@ class Engineer(Role): return False, rsp async def _think(self) -> Action | None: - if not self.src_workspace: - self.src_workspace = get_project_srcs_path(self.project_repo.workdir) - write_plan_and_change_filters = any_to_str_set([WriteTasks, FixBug]) - write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode]) - summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if not self.rc.news: return None msg = self.rc.news[0] + if msg.cause_by == any_to_str(UserRequirement): + self.rc.todo = PrepareDocuments( + key_descriptions={ + "project_path": 'the project path if exists in "Original Requirement"', + "src_filename": 'the file name of the source code file explicitly requested for modification if exists in "Original Requirement"', + }, + context=self.context, + send_to=any_to_str(self), + ) + return self.rc.todo + + if not self.src_workspace: + self.src_workspace = get_project_srcs_path(self.project_repo.workdir) + write_plan_and_change_filters = any_to_str_set([PrepareDocuments, WriteTasks, FixBug]) + write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if self.config.inc and msg.cause_by in write_plan_and_change_filters: logger.debug(f"TODO WriteCodePlanAndChange:{msg.model_dump_json()}") await self._new_code_plan_and_change_action(cause_by=msg.cause_by) @@ -308,7 +332,11 @@ class Engineer(Role): async def _new_code_actions(self): bug_fix = await self._is_fixbug() # Prepare file repos - changed_src_files = self.project_repo.srcs.all_files if bug_fix else self.project_repo.srcs.changed_files + changed_src_files = self.project_repo.srcs.changed_files + if self.context.kwargs.src_filename: + changed_src_files = {self.context.kwargs.src_filename: ChangeType.UNTRACTED} + if bug_fix: + changed_src_files = self.project_repo.srcs.all_files changed_task_files = self.project_repo.docs.task.changed_files changed_files = Documents() # Recode caused by upstream changes. @@ -319,6 +347,8 @@ class Engineer(Role): task_list = self._parse_tasks(task_doc) await self._init_python_folder(task_list) for task_filename in task_list: + if self.context.kwargs.src_filename and task_filename != self.context.kwargs.src_filename: + continue old_code_doc = await self.project_repo.srcs.get(task_filename) if not old_code_doc: old_code_doc = Document( diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 4eb1d249a..4beab5366 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -10,7 +10,7 @@ from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles.role import Role, RoleReactMode -from metagpt.utils.common import any_to_name +from metagpt.utils.common import any_to_name, any_to_str class ProductManager(Role): @@ -32,8 +32,8 @@ class ProductManager(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) - - self.set_actions([PrepareDocuments, WritePRD]) + self.enable_memory = False + self.set_actions([PrepareDocuments(send_to=any_to_str(self)), WritePRD]) self._watch([UserRequirement, PrepareDocuments]) self.rc.react_mode = RoleReactMode.BY_ORDER self.todo_action = any_to_name(WritePRD) @@ -47,6 +47,3 @@ class ProductManager(Role): self.config.git_reinit = False self.todo_action = any_to_name(WritePRD) return bool(self.rc.todo) - - async def _observe(self, ignore_memory=False) -> int: - return await super()._observe(ignore_memory=True) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 422d2889b..70bd3bf8b 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -6,9 +6,11 @@ @File : project_manager.py """ -from metagpt.actions import WriteTasks +from metagpt.actions import UserRequirement, WriteTasks from metagpt.actions.design_api import WriteDesign +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.roles.role import Role +from metagpt.utils.common import any_to_str class ProjectManager(Role): @@ -32,6 +34,21 @@ class ProjectManager(Role): def __init__(self, **kwargs) -> None: super().__init__(**kwargs) + self.enable_memory = False + self.set_actions([PrepareDocuments(send_to=any_to_str(self), context=self.context), WriteTasks]) + self._watch([UserRequirement, PrepareDocuments, WriteDesign]) - self.set_actions([WriteTasks]) - self._watch([WriteDesign]) + async def _think(self) -> bool: + """Decide what to do""" + mappings = { + any_to_str(UserRequirement): 0, + any_to_str(PrepareDocuments): 1, + any_to_str(WriteDesign): 1, + } + for i in self.rc.news: + idx = mappings.get(i.cause_by, -1) + if idx < 0: + continue + self.rc.todo = self.actions[idx] + return bool(self.rc.todo) + return False diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 04440c1cb..ed9c455a6 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -15,13 +15,19 @@ of SummarizeCode. """ -from metagpt.actions import DebugError, RunCode, WriteTest +from metagpt.actions import DebugError, RunCode, UserRequirement, WriteTest +from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.actions.summarize_code import SummarizeCode from metagpt.const import MESSAGE_ROUTE_TO_NONE from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Document, Message, RunCodeContext, TestingContext -from metagpt.utils.common import any_to_str_set, init_python_folder, parse_recipient +from metagpt.schema import AIMessage, Document, Message, RunCodeContext, TestingContext +from metagpt.utils.common import ( + any_to_str, + any_to_str_set, + init_python_folder, + parse_recipient, +) class QaEngineer(Role): @@ -37,19 +43,22 @@ class QaEngineer(Role): def __init__(self, **kwargs): super().__init__(**kwargs) + self.enable_memory = False # FIXME: a bit hack here, only init one action to circumvent _think() logic, # will overwrite _think() in future updates - self.set_actions([WriteTest]) - self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) + self.set_actions( + [ + WriteTest, + ] + ) + self._watch([UserRequirement, PrepareDocuments, SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 async def _write_test(self, message: Message) -> None: src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs - changed_files = set(src_file_repo.changed_files.keys()) - # Unit tests only. - if self.config.reqa_file and self.config.reqa_file not in changed_files: - changed_files.add(self.config.reqa_file) + reqa_file = self.context.kwargs.reqa_file or self.config.reqa_file + changed_files = {reqa_file} if reqa_file else set(src_file_repo.changed_files.keys()) for filename in changed_files: # write tests if not filename or "test" in filename: @@ -80,9 +89,8 @@ class QaEngineer(Role): additional_python_paths=[str(self.context.src_workspace)], ) self.publish_message( - Message( + AIMessage( content=run_code_context.model_dump_json(), - role=self.profile, cause_by=WriteTest, sent_from=self, send_to=self, @@ -116,9 +124,8 @@ class QaEngineer(Role): recipient = parse_recipient(result.summary) mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} self.publish_message( - Message( + AIMessage( content=run_code_context.model_dump_json(), - role=self.profile, cause_by=RunCode, sent_from=self, send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), @@ -131,9 +138,8 @@ class QaEngineer(Role): await self.project_repo.tests.save(filename=run_code_context.test_filename, content=code) run_code_context.output = None self.publish_message( - Message( + AIMessage( content=run_code_context.model_dump_json(), - role=self.profile, cause_by=DebugError, sent_from=self, send_to=self, @@ -141,18 +147,18 @@ class QaEngineer(Role): ) async def _act(self) -> Message: - await init_python_folder(self.project_repo.tests.workdir) + if self.project_path: + await init_python_folder(self.project_repo.tests.workdir) if self.test_round > self.test_round_allowed: - result_msg = Message( + result_msg = AIMessage( content=f"Exceeding {self.test_round_allowed} rounds of tests, skip (writing code counts as a round, too)", - role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg - code_filters = any_to_str_set({SummarizeCode}) + code_filters = any_to_str_set({PrepareDocuments, SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self.rc.news: @@ -167,16 +173,26 @@ class QaEngineer(Role): elif msg.cause_by in run_filters: # I ran my test code, time to fix bugs, if any await self._debug_error(msg) + elif msg.cause_by == any_to_str(UserRequirement): + return await self._parse_user_requirement(msg) self.test_round += 1 - return Message( + return AIMessage( content=f"Round {self.test_round} of tests done", - role=self.profile, cause_by=WriteTest, sent_from=self.profile, send_to=MESSAGE_ROUTE_TO_NONE, ) - async def _observe(self, ignore_memory=False) -> int: - # This role has events that trigger and execute themselves based on conditions, and cannot rely on the - # content of memory to activate. - return await super()._observe(ignore_memory=True) + async def _parse_user_requirement(self, msg: Message) -> AIMessage: + action = PrepareDocuments( + send_to=any_to_str(self), + key_descriptions={ + "project_path": 'the project path if exists in "Original Requirement"', + "reqa_file": 'the file name to rewrite unit test if exists in "Original Requirement"', + }, + context=self.context, + ) + rsp = await action.run([msg]) + if not self.src_workspace: + self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name + return rsp diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 00182106f..80a4d6744 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -34,7 +34,14 @@ from metagpt.context_mixin import ContextMixin from metagpt.logs import logger from metagpt.memory import Memory from metagpt.provider import HumanProvider -from metagpt.schema import Message, MessageQueue, SerializationMixin +from metagpt.schema import ( + AIMessage, + Message, + MessageQueue, + SerializationMixin, + Task, + TaskResult, +) from metagpt.strategy.planner import Planner from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator from metagpt.utils.project_repo import ProjectRepo @@ -134,6 +141,9 @@ class Role(SerializationMixin, ContextMixin, BaseModel): constraints: str = "" desc: str = "" is_human: bool = False + enable_memory: bool = ( + True # Stateless, atomic roles, or roles that use external storage can disable this to save memory. + ) role_id: str = "" states: list[str] = [] @@ -245,10 +255,9 @@ class Role(SerializationMixin, ContextMixin, BaseModel): return self def _init_action(self, action: Action): - if not action.private_config: - action.set_llm(self.llm, override=True) - else: - action.set_llm(self.llm, override=False) + action.set_context(self.context) + override = not action.private_config + action.set_llm(self.llm, override=override) action.set_prefix(self._get_prefix()) def set_action(self, action: Action): @@ -390,22 +399,22 @@ class Role(SerializationMixin, ContextMixin, BaseModel): logger.info(f"{self._setting}: to do {self.rc.todo}({self.rc.todo.name})") response = await self.rc.todo.run(self.rc.history) if isinstance(response, (ActionOutput, ActionNode)): - msg = Message( + msg = AIMessage( content=response.content, instruct_content=response.instruct_content, - role=self._setting, cause_by=self.rc.todo, sent_from=self, ) elif isinstance(response, Message): msg = response else: - msg = Message(content=response, role=self.profile, cause_by=self.rc.todo, sent_from=self) - self.rc.memory.add(msg) + msg = AIMessage(content=response, cause_by=self.rc.todo, sent_from=self) + if self.enable_memory: + self.rc.memory.add(msg) return msg - async def _observe(self, ignore_memory=False) -> int: + async def _observe(self) -> int: """Prepare new messages for processing from the message buffer and other sources.""" # Read unprocessed messages from the msg buffer. news = [] @@ -414,7 +423,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel): if not news: news = self.rc.msg_buffer.pop_all() # Store the read messages in your own memory to prevent duplicate processing. - old_messages = [] if ignore_memory else self.rc.memory.get() + old_messages = [] if not self.enable_memory else self.rc.memory.get() # Filter in messages of interest. self.rc.news = [ n for n in news if (n.cause_by in self.rc.watch or self.name in n.send_to) and n not in old_messages @@ -451,7 +460,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel): Use llm to select actions in _think dynamically """ actions_taken = 0 - rsp = Message(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act + rsp = AIMessage(content="No actions taken yet", cause_by=Action) # will be overwritten after Role _act while actions_taken < self.rc.max_react_loop: # think has_todo = await self._think() @@ -466,7 +475,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel): async def _act_by_order(self) -> Message: """switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...""" start_idx = self.rc.state if self.rc.state >= 0 else 0 # action to run from recovered state - rsp = Message(content="No actions taken yet") # return default message if actions=[] + rsp = AIMessage(content="No actions taken yet") # return default message if actions=[] for i in range(start_idx, len(self.states)): self._set_state(i) rsp = await self._act() @@ -523,6 +532,8 @@ class Role(SerializationMixin, ContextMixin, BaseModel): else: raise ValueError(f"Unsupported react mode: {self.rc.react_mode}") self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None + if isinstance(rsp, AIMessage): + rsp.with_agent(self._setting) return rsp def get_memories(self, k=0) -> list[Message]: diff --git a/metagpt/schema.py b/metagpt/schema.py index d0396ec26..9fa295211 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -38,6 +38,7 @@ from pydantic import ( ) from metagpt.const import ( + AGENT, MESSAGE_ROUTE_CAUSE_BY, MESSAGE_ROUTE_FROM, MESSAGE_ROUTE_TO, @@ -48,7 +49,7 @@ from metagpt.const import ( ) from metagpt.logs import ToolLogItem, log_tool_output, logger from metagpt.repo_parser import DotClassInfo -from metagpt.utils.common import any_to_str, any_to_str_set, import_class +from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, import_class from metagpt.utils.exceptions import handle_exception from metagpt.utils.serialize import ( actionoutout_schema_to_mapping, @@ -186,6 +187,14 @@ class Documents(BaseModel): return ActionOutput(content=self.model_dump_json(), instruct_content=self) +class Resource(BaseModel): + """Used by `Message`.`parse_resources`""" + + resource_type: str # the type of resource + value: str # a string type of resource content + description: str # explanation + + class Message(BaseModel): """list[: ]""" @@ -196,6 +205,7 @@ class Message(BaseModel): cause_by: str = Field(default="", validate_default=True) sent_from: str = Field(default="", validate_default=True) send_to: set[str] = Field(default={MESSAGE_ROUTE_TO_ALL}, validate_default=True) + metadata: Dict[str, str] = Field(default_factory=dict) # metadata for `content` and `instruct_content` @field_validator("id", mode="before") @classmethod @@ -311,14 +321,53 @@ class Message(BaseModel): logger.error(f"parse json failed: {val}, error:{err}") return None + async def parse_resources(self, llm: "BaseLLM", key_descriptions: Dict[str, str] = None) -> Dict: + """ + `parse_resources` corresponds to the in-context adaptation capability of the input of the atomic action, + which will be migrated to the context builder later. + + Args: + llm (BaseLLM): The instance of the BaseLLM class. + key_descriptions (Dict[str, str], optional): A dictionary containing descriptions for each key, + if provided. Defaults to None. + + Returns: + Dict: A dictionary containing parsed resources. + + """ + if not self.content: + return {} + content = f"## Original Requirement\n```text\n{self.content}\n```\n" + return_format = ( + "Return a markdown JSON object with:\n" + '- a "resources" key contain a list of objects. Each object with:\n' + ' - a "resource_type" key explain the type of resource;\n' + ' - a "value" key containing a string type of resource content;\n' + ' - a "description" key explaining why;\n' + ) + key_descriptions = key_descriptions or {} + for k, v in key_descriptions.items(): + return_format += f'- a "{k}" key containing {v};\n' + return_format += '- a "reason" key explaining why;\n' + instructions = ['Lists all the resources contained in the "Original Requirement".', return_format] + rsp = await llm.aask(msg=content, system_msgs=instructions) + json_data = CodeParser.parse_code(text=rsp, lang="json") + m = json.loads(json_data) + m["resources"] = [Resource(**i) for i in m.get("resources", [])] + return m + + def add_metadata(self, key: str, value: str): + self.metadata[key] = value + class UserMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ - def __init__(self, content: str): - super().__init__(content=content, role="user") + def __init__(self, content: str, **kwargs): + kwargs.pop("role", None) + super().__init__(content=content, role="user", **kwargs) class SystemMessage(Message): @@ -326,8 +375,9 @@ class SystemMessage(Message): Facilitate support for OpenAI messages """ - def __init__(self, content: str): - super().__init__(content=content, role="system") + def __init__(self, content: str, **kwargs): + kwargs.pop("role", None) + super().__init__(content=content, role="system", **kwargs) class AIMessage(Message): @@ -335,8 +385,17 @@ class AIMessage(Message): Facilitate support for OpenAI messages """ - def __init__(self, content: str): - super().__init__(content=content, role="assistant") + def __init__(self, content: str, **kwargs): + kwargs.pop("role", None) + super().__init__(content=content, role="assistant", **kwargs) + + def with_agent(self, name: str): + self.add_metadata(key=AGENT, value=name) + return self + + @property + def agent(self) -> str: + return self.metadata.get(AGENT, "") class Task(BaseModel): diff --git a/metagpt/software_company.py b/metagpt/software_company.py index f290d497a..431a3a179 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -7,6 +7,7 @@ from pathlib import Path import typer from metagpt.const import CONFIG_ROOT +from metagpt.utils.common import any_to_str from metagpt.utils.project_repo import ProjectRepo app = typer.Typer(add_completion=False, pretty_exceptions_show_locals=False) @@ -65,7 +66,7 @@ def generate_repo( idea = company.idea company.invest(investment) - company.run_project(idea) + company.run_project(idea, send_to=any_to_str(ProductManager)) asyncio.run(company.run(n_round=n_round)) return ctx.repo diff --git a/metagpt/strategy/tot.py b/metagpt/strategy/tot.py index 88c2ac9ff..17ce63211 100644 --- a/metagpt/strategy/tot.py +++ b/metagpt/strategy/tot.py @@ -62,7 +62,7 @@ class ThoughtSolverBase(BaseModel): current_state=current_state, **{"n_generate_sample": self.config.n_generate_sample} ) rsp = await self.llm.aask(msg=state_prompt + "\n" + OUTPUT_FORMAT) - thoughts = CodeParser.parse_code(block="", text=rsp) + thoughts = CodeParser.parse_code(text=rsp) thoughts = eval(thoughts) # fixme 避免不跟随，生成过多nodes # valid_thoughts = [_node for idx, _node in enumerate(thoughts) if idx < self.n_generate_sample] diff --git a/metagpt/team.py b/metagpt/team.py index 79c4c36aa..c3498b96b 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -20,7 +20,7 @@ from metagpt.context import Context from metagpt.environment import Environment from metagpt.logs import logger from metagpt.roles import Role -from metagpt.schema import Message +from metagpt.schema import UserMessage from metagpt.utils.common import ( NoMoneyException, read_json_file, @@ -102,7 +102,7 @@ class Team(BaseModel): # Human requirement. self.env.publish_message( - Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL), + UserMessage(content=idea, cause_by=UserRequirement, send_to=send_to or MESSAGE_ROUTE_TO_ALL), peekable=False, ) diff --git a/metagpt/tools/libs/__init__.py b/metagpt/tools/libs/__init__.py index 4533f03fb..f4a112a69 100644 --- a/metagpt/tools/libs/__init__.py +++ b/metagpt/tools/libs/__init__.py @@ -17,9 +17,6 @@ from metagpt.tools.libs import ( deployer, ) from metagpt.tools.libs.env import get_env, set_get_env_entry, default_get_env, get_env_description -from metagpt.tools.libs.software_development import ( - git_archive, -) _ = ( data_preprocess, @@ -28,7 +25,6 @@ _ = ( gpt_v_generator, web_scraping, email_login, - git_archive, terminal, file_manager, browser, diff --git a/metagpt/tools/libs/gpt_v_generator.py b/metagpt/tools/libs/gpt_v_generator.py index 4eba3d5ee..baedc3d61 100644 --- a/metagpt/tools/libs/gpt_v_generator.py +++ b/metagpt/tools/libs/gpt_v_generator.py @@ -89,7 +89,7 @@ class GPTvGenerator: webpages_path.mkdir(parents=True, exist_ok=True) index_path = webpages_path / "index.html" - index_path.write_text(CodeParser.parse_code(block=None, text=webpages, lang="html")) + index_path.write_text(CodeParser.parse_code(text=webpages, lang="html")) extract_and_save_code(folder=webpages_path, text=webpages, pattern="styles?.css", language="css") @@ -102,5 +102,5 @@ def extract_and_save_code(folder, text, pattern, language): word = re.search(pattern, text) if word: path = folder / word.group(0) - code = CodeParser.parse_code(block=None, text=text, lang=language) + code = CodeParser.parse_code(text=text, lang=language) path.write_text(code, encoding="utf-8") diff --git a/metagpt/tools/libs/software_development.py b/metagpt/tools/libs/software_development.py index 0149e7e35..f250bcd2d 100644 --- a/metagpt/tools/libs/software_development.py +++ b/metagpt/tools/libs/software_development.py @@ -3,384 +3,10 @@ from __future__ import annotations from pathlib import Path -from typing import Optional -from metagpt.const import ASSISTANT_ALIAS, BUGFIX_FILENAME, REQUIREMENT_FILENAME +from metagpt.const import ASSISTANT_ALIAS from metagpt.logs import ToolLogItem, log_tool_output -from metagpt.schema import BugFixContext, Message from metagpt.tools.tool_registry import register_tool -from metagpt.utils.common import any_to_str -from metagpt.utils.project_repo import ProjectRepo - - -@register_tool(tags=["software development", "ProductManager"]) -async def write_prd(idea: str, project_path: Optional[str | Path] = None) -> Path: - """Writes a PRD based on user requirements. - - Args: - idea (str): The idea or concept for the PRD. - project_path (Optional[str|Path], optional): The path to an existing project directory. - If it's None, a new project path will be created. Defaults to None. - - Returns: - Path: The path to the PRD files under the project directory - - Example: - >>> # Create a new project: - >>> from metagpt.tools.libs.software_development import write_prd - >>> prd_path = await write_prd("Create a new feature for the application") - >>> print(prd_path) - '/path/to/project_path/docs/prd/' - - >>> # Add user requirements to the exists project: - >>> from metagpt.tools.libs.software_development import write_prd - >>> project_path = '/path/to/exists_project_path' - >>> prd_path = await write_prd("Create a new feature for the application", project_path=project_path) - >>> print(prd_path = ) - '/path/to/project_path/docs/prd/' - """ - from metagpt.actions import UserRequirement - from metagpt.context import Context - from metagpt.roles import ProductManager - - log_tool_output(output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_prd.__name__)], tool_name=write_prd.__name__) - - ctx = Context() - if project_path and Path(project_path).exists(): - ctx.config.project_path = Path(project_path) - ctx.config.inc = True - - role = ProductManager(context=ctx) - msg = await role.run(with_message=Message(content=idea, cause_by=UserRequirement)) - await role.run(with_message=msg) - - outputs = [ - ToolLogItem(name="Intermedia PRD File", value=str(ctx.repo.docs.prd.workdir / i)) - for i in ctx.repo.docs.prd.changed_files.keys() - ] - outputs.extend( - [ - ToolLogItem(name="PRD File", value=str(ctx.repo.resources.prd.workdir / i)) - for i in ctx.repo.resources.prd.changed_files.keys() - ] - ) - outputs.extend( - [ - ToolLogItem(name="Competitive Analysis", value=str(ctx.repo.resources.competitive_analysis.workdir / i)) - for i in ctx.repo.resources.competitive_analysis.changed_files.keys() - ] - ) - log_tool_output(output=outputs, tool_name=write_prd.__name__) - - return ctx.repo.docs.prd.workdir - - -@register_tool(tags=["Design", "software development", "Architect"]) -async def write_design(prd_path: str | Path) -> Path: - """Writes a system design to the project repository, based on the PRD of the project. - - Args: - prd_path (str|Path): The path to the PRD files under the project directory. - - Returns: - Path: The path to the system design files under the project directory. - - Example: - >>> from metagpt.tools.libs.software_development import write_design - >>> prd_path = '/path/to/project_path/docs/prd' # Returned by `write_prd` - >>> system_design_path = await write_desgin(prd_path) - >>> print(system_design_path) - '/path/to/project_path/docs/system_design/' - - """ - from metagpt.actions import WritePRD - from metagpt.context import Context - from metagpt.roles import Architect - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_design.__name__)], tool_name=write_design.__name__ - ) - - ctx = Context() - prd_path = Path(prd_path) - project_path = (Path(prd_path) if not prd_path.is_file() else prd_path.parent) / "../.." - ctx.set_repo_dir(project_path) - - role = Architect(context=ctx) - await role.run(with_message=Message(content="", cause_by=WritePRD)) - - outputs = [ - ToolLogItem(name="Intermedia Design File", value=str(ctx.repo.docs.system_design.workdir / i)) - for i in ctx.repo.docs.system_design.changed_files.keys() - ] - for i in ctx.repo.resources.system_design.changed_files.keys(): - outputs.append(ToolLogItem(name="Design File", value=str(ctx.repo.resources.system_design.workdir / i))) - for i in ctx.repo.resources.data_api_design.changed_files.keys(): - outputs.append( - ToolLogItem(name="Class Diagram File", value=str(ctx.repo.resources.data_api_design.workdir / i)) - ) - for i in ctx.repo.resources.seq_flow.changed_files.keys(): - outputs.append(ToolLogItem(name="Sequence Diagram File", value=str(ctx.repo.resources.seq_flow.workdir / i))) - log_tool_output(output=outputs, tool_name=write_design.__name__) - - return ctx.repo.docs.system_design.workdir - - -@register_tool(tags=["software development", "Architect"]) -async def write_project_plan(system_design_path: str | Path) -> Path: - """Writes a project plan to the project repository, based on the design of the project. - - Args: - system_design_path (str|Path): The path to the system design files under the project directory. - - Returns: - Path: The path to task files under the project directory. - - Example: - >>> from metagpt.tools.libs.software_development import write_project_plan - >>> system_design_path = '/path/to/project_path/docs/system_design/' # Returned by `write_design` - >>> task_path = await write_project_plan(system_design_path) - >>> print(task_path) - '/path/to/project_path/docs/task' - - """ - from metagpt.actions import WriteDesign - from metagpt.context import Context - from metagpt.roles import ProjectManager - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_project_plan.__name__)], - tool_name=write_project_plan.__name__, - ) - - ctx = Context() - system_design_path = Path(system_design_path) - project_path = (system_design_path if not system_design_path.is_file() else system_design_path.parent) / "../.." - ctx.set_repo_dir(project_path) - - role = ProjectManager(context=ctx) - await role.run(with_message=Message(content="", cause_by=WriteDesign)) - - outputs = [ - ToolLogItem(name="Intermedia Project Plan", value=str(ctx.repo.docs.task.workdir / i)) - for i in ctx.repo.docs.task.changed_files.keys() - ] - outputs.extend( - [ - ToolLogItem(name="Project Plan", value=str(ctx.repo.resources.api_spec_and_task.workdir / i)) - for i in ctx.repo.resources.api_spec_and_task.changed_files.keys() - ] - ) - log_tool_output(output=outputs, tool_name=write_project_plan.__name__) - - return ctx.repo.docs.task.workdir - - -@register_tool(tags=["software development", "Engineer"]) -async def write_codes(task_path: str | Path, inc: bool = False) -> Path: - """Writes code to implement designed features according to the project plan and adds them to the project repository. - In code writing tasks, prioritize calling this tool against writing code from scratch directly. - - Args: - task_path (str|Path): The path to task files under the project directory. - inc (bool, optional): Whether to write incremental codes. Defaults to False. - - Returns: - Path: The path to the source code files under the project directory. - - Example: - # Write codes to a new project - >>> from metagpt.tools.libs.software_development import write_codes - >>> task_path = '/path/to/project_path/docs/task' # Returned by `write_project_plan` - >>> src_path = await write_codes(task_path) - >>> print(src_path) - '/path/to/project_path/src/' - - # Write increment codes to the exists project - >>> from metagpt.tools.libs.software_development import write_codes - >>> task_path = '/path/to/project_path/docs/task' # Returned by `write_prd` - >>> src_path = await write_codes(task_path, inc=True) - >>> print(src_path) - '/path/to/project_path/src/' - """ - from metagpt.actions import WriteTasks - from metagpt.context import Context - from metagpt.roles import Engineer - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=write_codes.__name__)], tool_name=write_codes.__name__ - ) - - ctx = Context() - ctx.config.inc = inc - task_path = Path(task_path) - project_path = (task_path if not task_path.is_file() else task_path.parent) / "../.." - ctx.set_repo_dir(project_path) - - role = Engineer(context=ctx) - msg = Message(content="", cause_by=WriteTasks, send_to=role) - me = {any_to_str(role), role.name} - while me.intersection(msg.send_to): - msg = await role.run(with_message=msg) - - outputs = [ - ToolLogItem(name="Source File", value=str(ctx.repo.srcs.workdir / i)) - for i in ctx.repo.srcs.changed_files.keys() - ] - log_tool_output(output=outputs, tool_name=write_codes.__name__) - - return ctx.repo.srcs.workdir - - -@register_tool(tags=["software development", "QaEngineer"]) -async def run_qa_test(src_path: str | Path) -> Path: - """Run QA test on the project repository. - - Args: - src_path (str|Path): The path to the source code files under the project directory. - - Returns: - Path: The path to the unit tests under the project directory - - Example: - >>> from metagpt.tools.libs.software_development import run_qa_test - >>> src_path = '/path/to/project_path/src/' # Returned by `write_codes` - >>> test_path = await run_qa_test(src_path) - >>> print(test_path) - '/path/to/project_path/tests' - """ - from metagpt.actions.summarize_code import SummarizeCode - from metagpt.context import Context - from metagpt.environment import Environment - from metagpt.roles import QaEngineer - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=run_qa_test.__name__)], tool_name=run_qa_test.__name__ - ) - - ctx = Context() - src_path = Path(src_path) - project_path = (src_path if not src_path.is_file() else src_path.parent) / ".." - ctx.set_repo_dir(project_path) - ctx.src_workspace = ctx.git_repo.workdir / ctx.git_repo.workdir.name - - env = Environment(context=ctx) - role = QaEngineer(context=ctx) - env.add_role(role) - - msg = Message(content="", cause_by=SummarizeCode, send_to=role) - env.publish_message(msg) - - while not env.is_idle: - await env.run() - - outputs = [ - ToolLogItem(name="Unit Test File", value=str(ctx.repo.tests.workdir / i)) - for i in ctx.repo.tests.changed_files.keys() - ] - log_tool_output(output=outputs, tool_name=run_qa_test.__name__) - - return ctx.repo.tests.workdir - - -@register_tool(tags=["software development", "Engineer"]) -async def fix_bug(project_path: str | Path, issue: str) -> Path: - """Fix bugs in the project repository. - - Args: - project_path (str|Path): The path to the project repository. - issue (str): Description of the bug or issue. - - Returns: - Path: The path to the project directory - - Example: - >>> from metagpt.tools.libs.software_development import fix_bug - >>> project_path = '/path/to/project_path' # Returned by `write_codes` - >>> issue = 'Exception: exception about ...; Bug: bug about ...; Issue: issue about ...' - >>> project_path = await fix_bug(project_path=project_path, issue=issue) - >>> print(project_path) - '/path/to/project_path' - """ - from metagpt.actions.fix_bug import FixBug - from metagpt.context import Context - from metagpt.roles import Engineer - - log_tool_output(output=[ToolLogItem(name=ASSISTANT_ALIAS, value=fix_bug.__name__)], tool_name=fix_bug.__name__) - - ctx = Context() - ctx.set_repo_dir(project_path) - ctx.src_workspace = ctx.git_repo.workdir / ctx.git_repo.workdir.name - await ctx.repo.docs.save(filename=BUGFIX_FILENAME, content=issue) - await ctx.repo.docs.save(filename=REQUIREMENT_FILENAME, content="") - - role = Engineer(context=ctx) - bug_fix = BugFixContext(filename=BUGFIX_FILENAME) - msg = Message( - content=bug_fix.model_dump_json(), - instruct_content=bug_fix, - role="", - cause_by=FixBug, - sent_from=role, - send_to=role, - ) - me = {any_to_str(role), role.name} - while me.intersection(msg.send_to): - msg = await role.run(with_message=msg) - - outputs = [ - ToolLogItem(name="Changed File", value=str(ctx.repo.srcs.workdir / i)) - for i in ctx.repo.srcs.changed_files.keys() - ] - log_tool_output(output=outputs, tool_name=fix_bug.__name__) - - return project_path - - -@register_tool(tags=["software development", "git"]) -async def git_archive(project_path: str | Path) -> str: - """Stage and commit changes for the project repository using Git. - - Args: - project_path (str|Path): The path to the project repository. - - - Returns: - git log - - Example: - >>> from metagpt.tools.libs.software_development import git_archive - >>> project_path = '/path/to/project_path' # Returned by `write_prd` - >>> git_log = await git_archive(project_path=project_path) - >>> print(git_log) - commit a221d1c418c07f2b4fc07001e486285ead1a520a (HEAD -> feature/toollib/software_company, geekan/main) - Merge: e01afd09 4a72f398 - Author: Sirui Hong - Date: Tue Mar 19 15:16:03 2024 +0800 - Merge pull request #1037 from iorisa/fixbug/issues/1018 - fixbug: #1018 - - """ - from metagpt.context import Context - - log_tool_output( - output=[ToolLogItem(name=ASSISTANT_ALIAS, value=git_archive.__name__)], tool_name=git_archive.__name__ - ) - - ctx = Context() - project_dir = ProjectRepo.search_project_path(project_path) - if not project_dir: - ValueError(f"{project_path} is not a valid git repository.") - ctx.set_repo_dir(project_dir) - files = " ".join(ctx.git_repo.changed_files.keys()) - outputs = [ToolLogItem(name="cmd", value=f"git add {files}")] - log_tool_output(output=outputs, tool_name=git_archive.__name__) - ctx.git_repo.archive() - - outputs = [ToolLogItem(name="cmd", value="git commit -m 'Archive'")] - log_tool_output(output=outputs, tool_name=git_archive.__name__) - - return ctx.git_repo.log() @register_tool(tags=["software development", "import git repo"]) diff --git a/metagpt/tools/tool_recommend.py b/metagpt/tools/tool_recommend.py index 01ff61834..05e8e1400 100644 --- a/metagpt/tools/tool_recommend.py +++ b/metagpt/tools/tool_recommend.py @@ -132,7 +132,7 @@ class ToolRecommender(BaseModel): topk=topk, ) rsp = await LLM().aask(prompt, stream=False) - rsp = CodeParser.parse_code(block=None, text=rsp) + rsp = CodeParser.parse_code(text=rsp) ranked_tools = json.loads(rsp) valid_tools = validate_tool_names(ranked_tools) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index ffc25ac05..384d4e8ac 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -271,7 +271,7 @@ class CodeParser: return block_dict @classmethod - def parse_code(cls, block: Optional[str], text: str, lang: str = "") -> str: + def parse_code(cls, text: str, lang: str = "", block: Optional[str] = None) -> str: if block: text = cls.parse_block(block, text) pattern = rf"```{lang}.*?\s+(.*?)```" @@ -287,7 +287,7 @@ class CodeParser: @classmethod def parse_str(cls, block: str, text: str, lang: str = ""): - code = cls.parse_code(block, text, lang) + code = cls.parse_code(block=block, text=text, lang=lang) code = code.split("=")[-1] code = code.strip().strip("'").strip('"') return code @@ -295,7 +295,7 @@ class CodeParser: @classmethod def parse_file_list(cls, block: str, text: str, lang: str = "") -> list[str]: # Regular expression pattern to find the tasks list. - code = cls.parse_code(block, text, lang) + code = cls.parse_code(block=block, text=text, lang=lang) # print(code) pattern = r"\s*(.*=.*)?(\[.*\])" diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 1709e1f5b..42623f807 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -110,7 +110,7 @@ async def test_write_refined_code(context, git_dir): # old_workspace contains the legacy code await context.repo.with_src_path(context.repo.old_workspace).srcs.save( - filename="game.py", content=CodeParser.parse_code(block="", text=REFINED_CODE_INPUT_SAMPLE) + filename="game.py", content=CodeParser.parse_code(text=REFINED_CODE_INPUT_SAMPLE) ) ccontext = CodingContext( diff --git a/tests/metagpt/actions/test_write_code_plan_and_change_an.py b/tests/metagpt/actions/test_write_code_plan_and_change_an.py index 5c262b4b7..5bc860469 100644 --- a/tests/metagpt/actions/test_write_code_plan_and_change_an.py +++ b/tests/metagpt/actions/test_write_code_plan_and_change_an.py @@ -45,7 +45,7 @@ async def test_write_code_plan_and_change_an(mocker, context, git_dir): await context.repo.docs.task.save(filename="2.json", content=json.dumps(REFINED_TASK_JSON)) await context.repo.with_src_path(context.repo.old_workspace).srcs.save( - filename="game.py", content=CodeParser.parse_code(block="", text=REFINED_CODE_INPUT_SAMPLE) + filename="game.py", content=CodeParser.parse_code(text=REFINED_CODE_INPUT_SAMPLE) ) root = ActionNode.from_children( diff --git a/tests/metagpt/roles/test_engineer.py b/tests/metagpt/roles/test_engineer.py index d263a8a2f..d5eae662f 100644 --- a/tests/metagpt/roles/test_engineer.py +++ b/tests/metagpt/roles/test_engineer.py @@ -91,7 +91,7 @@ target_code = """task_list = [ def test_parse_code(): - code = CodeParser.parse_code("Task list", TASKS, lang="python") + code = CodeParser.parse_code(block="Task list", text=TASKS, lang="python") logger.info(code) assert isinstance(code, str) assert target_code == code diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index 7559655d3..07f5b4305 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -11,17 +11,44 @@ from pathlib import Path import pytest from metagpt.actions import UserRequirement +from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.context import Context from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.roles import Architect, ProductManager, Role -from metagpt.schema import Message +from metagpt.roles import ( + Architect, + Engineer, + ProductManager, + ProjectManager, + QaEngineer, + Role, +) +from metagpt.schema import Message, UserMessage +from metagpt.utils.common import any_to_str, is_send_to serdeser_path = Path(__file__).absolute().parent.joinpath("../data/serdeser_storage") +class MockEnv(Environment): + def publish_message(self, message: Message, peekable: bool = True) -> bool: + consumers = [] + for role, addrs in self.member_addrs.items(): + if is_send_to(message, addrs): + role.put_message(message) + consumers.append(role) + if not consumers: + logger.warning(f"Message no recipients: {message.dump()}") + if message.cause_by in [any_to_str(UserRequirement), any_to_str(PrepareDocuments)]: + assert len(consumers) == 1 + + return True + + @pytest.fixture def env(): - return Environment() + context = Context() + context.kwargs.tag = __file__ + return MockEnv(context=context) def test_add_role(env: Environment): @@ -54,10 +81,56 @@ async def test_publish_and_process_message(env: Environment): env.add_roles([product_manager, architect]) - env.publish_message(Message(role="User", content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement)) + env.publish_message(UserMessage(content="需要一个基于LLM做总结的搜索引擎", cause_by=UserRequirement, send_to=product_manager)) await env.run(k=2) - logger.info(f"{env.history=}") - assert len(env.history) > 10 + logger.info(f"{env.history}") + assert len(env.history.storage) == 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("content", "send_to"), + [ + ("snake game", any_to_str(ProductManager)), + ( + "Rewrite the PRD file of the project at '/Users/iorishinier/github/MetaGPT/workspace/snake_game', add 'moving enemy' to the original requirement", + any_to_str(ProductManager), + ), + ( + "Add 'random moving enemy, and dispears after 10 seconds' design to the project at '/Users/iorishinier/github/MetaGPT/workspace/snake_game'", + any_to_str(Architect), + ), + ( + 'Rewrite the tasks file of the project at "/Users/iorishinier/github/MetaGPT/workspace/snake_game"', + any_to_str(ProjectManager), + ), + ( + "Rewrite 'main.py' of the project at '/Users/iorishinier/github/MetaGPT/workspace/snake_game'", + any_to_str(Engineer), + ), + ( + "Rewrite the unit test of 'main.py' at '/Users/iorishinier/github/MetaGPT/workspace/snake_game'", + any_to_str(QaEngineer), + ), + ], +) +async def test_env(content, send_to): + context = Context() + env = MockEnv(context=context) + env.add_roles( + [ + ProductManager(context=context), + Architect(context=context), + ProjectManager(context=context), + Engineer(n_borg=5, use_code_review=True, context=context), + QaEngineer(context=context, test_round_allowed=2), + ] + ) + msg = UserMessage(content=content, send_to=send_to) + env.publish_message(msg) + while not env.is_idle: + await env.run() + pass if __name__ == "__main__": diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 22f6ae9fb..6f54b062d 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -350,5 +350,47 @@ class TestPlan: assert plan.current_task_id == "2" +@pytest.mark.parametrize( + ("content", "key_descriptions"), + [ + ( + """ +Traceback (most recent call last): + File "/Users/iorishinier/github/MetaGPT/workspace/game_2048_1/game_2048/main.py", line 38, in + Main().main() + File "/Users/iorishinier/github/MetaGPT/workspace/game_2048_1/game_2048/main.py", line 28, in main + self.user_interface.draw() + File "/Users/iorishinier/github/MetaGPT/workspace/game_2048_1/game_2048/user_interface.py", line 16, in draw + if grid[i][j] != 0: +TypeError: 'Grid' object is not subscriptable + """, + { + "filename": "the string type of the path name of the source code where the bug resides", + "line": "the integer type of the line error occurs", + "function_name": "the string type of the function name the error occurs in", + "code": "the string type of the codes where the error occurs at", + "info": "the string type of the error information", + }, + ), + ( + "将代码提交到github上的iorisa/repo1的branch1分支，发起pull request ，合并到master分支。", + { + "repo_name": "the string type of github repo to create pull", + "head": "the string type of github branch to be pushed", + "base": "the string type of github branch to merge the changes into", + }, + ), + ], +) +async def test_parse_resources(context, content: str, key_descriptions): + msg = Message(content=content) + llm = context.llm_with_cost_manager_from_llm_config(context.config.llm) + result = await msg.parse_resources(llm=llm, key_descriptions=key_descriptions) + assert result + assert result.get("resources") + for k in key_descriptions.keys(): + assert k in result + + if __name__ == "__main__": pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_code_parser.py b/tests/metagpt/utils/test_code_parser.py index 294324b8f..f4d822f85 100644 --- a/tests/metagpt/utils/test_code_parser.py +++ b/tests/metagpt/utils/test_code_parser.py @@ -119,7 +119,7 @@ class TestCodeParser: assert "game.py" in result def test_parse_code(self, parser, text): - result = parser.parse_code("Task list", text, "python") + result = parser.parse_code(block="Task list", text=text, lang="python") print(result) assert "game.py" in result diff --git a/tests/mock/mock_llm.py b/tests/mock/mock_llm.py index f6c206d5e..168125448 100644 --- a/tests/mock/mock_llm.py +++ b/tests/mock/mock_llm.py @@ -85,7 +85,7 @@ class MockLLM(OriginalLLM): format_msgs: Optional[list[dict[str, str]]] = None, images: Optional[Union[str, list[str]]] = None, timeout=LLM_API_TIMEOUT, - stream=True, + stream=False, ) -> str: # used to identify it a message has been called before if isinstance(msg, list):