diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5fd538720..b760c96d8 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -22,7 +22,6 @@ from metagpt.schema import ( SerializationMixin, TestingContext, ) -from metagpt.utils.project_repo import ProjectRepo class Action(SerializationMixin, ContextMixin, BaseModel): @@ -36,12 +35,6 @@ class Action(SerializationMixin, ContextMixin, BaseModel): desc: str = "" # for skill manager node: ActionNode = Field(default=None, exclude=True) - @property - def repo(self) -> ProjectRepo: - if not self.context.repo: - self.context.repo = ProjectRepo(self.context.git_repo) - return self.context.repo - @property def prompt_schema(self): return self.config.prompt_schema diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index b027616f7..8f0f52266 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -9,13 +9,15 @@ 2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. """ import re +from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import RunCodeContext, RunCodeResult from metagpt.utils.common import CodeParser +from metagpt.utils.project_repo import ProjectRepo PROMPT_TEMPLATE = """ NOTICE @@ -47,6 +49,8 @@ Now you should start rewriting the code: class DebugError(Action): i_context: RunCodeContext = Field(default_factory=RunCodeContext) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) async def run(self, *args, **kwargs) -> str: output_doc = await self.repo.test_outputs.get(filename=self.i_context.output_filename) @@ -59,9 +63,7 @@ class DebugError(Action): return "" logger.info(f"Debug and rewrite {self.i_context.test_filename}") - code_doc = await self.repo.with_src_path(self.context.src_workspace).srcs.get( - filename=self.i_context.code_filename - ) + code_doc = await self.repo.srcs.get(filename=self.i_context.code_filename) if not code_doc: return "" test_doc = await self.repo.tests.get(filename=self.i_context.test_filename) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 613c4a47b..2e84cc463 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -13,6 +13,8 @@ import json from pathlib import Path from typing import Optional +from pydantic import BaseModel, Field + from metagpt.actions import Action from metagpt.actions.design_api_an import ( DATA_STRUCTURES_AND_INTERFACES, @@ -26,6 +28,7 @@ from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO from metagpt.logs import logger from metagpt.schema import AIMessage, Document, Documents, Message from metagpt.utils.mermaid import mermaid_to_file +from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import DocsReporter, GalleryReporter NEW_REQ_TEMPLATE = """ @@ -45,21 +48,25 @@ class WriteDesign(Action): "data structures, library tables, processes, and paths. Please provide your design, feedback " "clearly and in detail." ) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) async def run(self, with_messages: Message, schema: str = None): - # Use `git status` to identify which PRD documents have been modified in the `docs/prd` directory. - changed_prds = self.repo.docs.prd.changed_files - # Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone - # changes. - changed_system_designs = self.repo.docs.system_design.changed_files + self.input_args = with_messages[0].instruct_content + self.repo = ProjectRepo(self.input_args.project_path) + changed_prds = self.input_args.changed_prd_filenames + changed_system_designs = [ + str(self.repo.docs.system_design.workdir / i) + for i in list(self.repo.docs.system_design.changed_files.keys()) + ] # For those PRDs and design documents that have undergone changes, regenerate the design content. changed_files = Documents() - for filename in changed_prds.keys(): + for filename in changed_prds: doc = await self._update_system_design(filename=filename) changed_files.docs[filename] = doc - for filename in changed_system_designs.keys(): + for filename in changed_system_designs: if filename in changed_files.docs: continue doc = await self._update_system_design(filename=filename) @@ -68,6 +75,11 @@ class WriteDesign(Action): logger.info("Nothing has changed.") # Wait until all files under `docs/system_designs/` are processed before sending the publish message, # leaving room for global optimization in subsequent steps. + kvs = self.input_args.model_dump() + kvs["changed_system_design_filenames"] = [ + str(self.repo.docs.system_design.workdir / i) + for i in list(self.repo.docs.system_design.changed_files.keys()) + ] return AIMessage( content="Designing is complete. " + "\n".join( @@ -75,6 +87,7 @@ class WriteDesign(Action): + list(self.repo.resources.data_api_design.changed_files.keys()) + list(self.repo.resources.seq_flow.changed_files.keys()) ), + instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteDesignOutput"), cause_by=self, ) @@ -89,14 +102,15 @@ class WriteDesign(Action): return system_design_doc async def _update_system_design(self, filename) -> Document: - prd = await self.repo.docs.prd.get(filename) - old_system_design_doc = await self.repo.docs.system_design.get(filename) + root_relative_path = Path(filename).relative_to(self.repo.workdir) + prd = await Document.load(filename=filename, project_path=self.repo.workdir) + old_system_design_doc = await self.repo.docs.system_design.get(root_relative_path.name) async with DocsReporter(enable_llm_stream=True) as reporter: await reporter.async_report({"type": "design"}, "meta") if not old_system_design_doc: system_design = await self._new_system_design(context=prd.content) doc = await self.repo.docs.system_design.save( - filename=filename, + filename=prd.filename, content=system_design.instruct_content.model_dump_json(), dependencies={prd.root_relative_path}, ) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index eb674374c..89ebd59a3 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -17,6 +17,7 @@ from metagpt.logs import logger from metagpt.schema import AIMessage from metagpt.utils.common import any_to_str from metagpt.utils.file_repository import FileRepository +from metagpt.utils.project_repo import ProjectRepo class PrepareDocuments(Action): @@ -36,7 +37,7 @@ class PrepareDocuments(Action): def config(self): return self.context.config - def _init_repo(self): + def _init_repo(self) -> ProjectRepo: """Initialize the Git environment.""" if not self.config.project_path: name = self.config.project_name or FileRepository.new_filename() @@ -47,6 +48,7 @@ class PrepareDocuments(Action): shutil.rmtree(path) self.config.project_path = path self.context.set_repo_dir(path) + return ProjectRepo(path) async def run(self, with_messages, **kwargs): """Create and initialize the workspace folder, initialize the Git environment.""" @@ -67,10 +69,22 @@ class PrepareDocuments(Action): max_auto_summarize_code=0, ) - self._init_repo() + repo = self._init_repo() # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. - doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content) + await repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content) # Send a Message notification to the WritePRD action, instructing it to process requirements using # `docs/requirement.txt` and `docs/prd/`. - return AIMessage(content="", instruct_content=doc, cause_by=self, send_to=self.send_to) + return AIMessage( + content="", + instruct_content=AIMessage.create_instruct_value( + kvs={ + "project_path": str(repo.workdir), + "requirements_filename": str(repo.docs.workdir / REQUIREMENT_FILENAME), + "prd_filenames": [str(repo.docs.prd.workdir / i) for i in repo.docs.prd.all_files], + }, + class_name="PrepareDocumentsOutput", + ), + cause_by=self, + send_to=self.send_to, + ) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index ef0fe6fc6..55356f58b 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -11,13 +11,17 @@ """ import json +from pathlib import Path from typing import Optional +from pydantic import BaseModel, Field + from metagpt.actions.action import Action from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import AIMessage, Document, Documents +from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import DocsReporter NEW_REQ_TEMPLATE = """ @@ -32,10 +36,14 @@ NEW_REQ_TEMPLATE = """ class WriteTasks(Action): name: str = "CreateTasks" i_context: Optional[str] = None + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) async def run(self, with_messages): - changed_system_designs = self.repo.docs.system_design.changed_files - changed_tasks = self.repo.docs.task.changed_files + self.input_args = with_messages[0].instruct_content + self.repo = ProjectRepo(self.input_args.project_path) + changed_system_designs = self.input_args.changed_system_design_filenames + changed_tasks = [str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())] change_files = Documents() # Rewrite the system designs that have undergone changes based on the git head diff under # `docs/system_designs/`. @@ -54,6 +62,11 @@ class WriteTasks(Action): logger.info("Nothing has changed.") # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for # global optimization in subsequent steps. + kvs = self.input_args.model_dump() + kvs["changed_task_filenames"] = [ + str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys()) + ] + kvs["python_package_dependency_filename"] = str(self.repo.workdir / PACKAGE_REQUIREMENTS_FILENAME) return AIMessage( content="WBS is completed. " + "\n".join( @@ -61,12 +74,14 @@ class WriteTasks(Action): + list(self.repo.docs.task.changed_files.keys()) + list(self.repo.resources.api_spec_and_task.changed_files.keys()) ), + instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTaskOutput"), cause_by=self, ) async def _update_tasks(self, filename): - system_design_doc = await self.repo.docs.system_design.get(filename) - task_doc = await self.repo.docs.task.get(filename) + root_relative_path = Path(filename).relative_to(self.repo.workdir) + system_design_doc = await Document.load(filename=filename, project_path=self.repo.workdir) + task_doc = await self.repo.docs.task.get(root_relative_path.name) async with DocsReporter(enable_llm_stream=True) as reporter: await reporter.async_report({"type": "task"}, "meta") if task_doc: @@ -75,7 +90,7 @@ class WriteTasks(Action): else: rsp = await self._run_new_tasks(context=system_design_doc.content) task_doc = await self.repo.docs.task.save( - filename=filename, + filename=system_design_doc.filename, content=rsp.instruct_content.model_dump_json(), dependencies={system_design_doc.root_relative_path}, ) diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index d21b62f83..e3556caa7 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -6,13 +6,16 @@ @Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ from pathlib import Path +from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.logs import logger from metagpt.schema import CodeSummarizeContext +from metagpt.utils.common import get_markdown_code_block_type +from metagpt.utils.project_repo import ProjectRepo PROMPT_TEMPLATE = """ NOTICE @@ -90,6 +93,8 @@ flowchart TB class SummarizeCode(Action): name: str = "SummarizeCode" i_context: CodeSummarizeContext = Field(default_factory=CodeSummarizeContext) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60)) async def summarize_code(self, prompt): @@ -101,11 +106,10 @@ class SummarizeCode(Action): design_doc = await self.repo.docs.system_design.get(filename=design_pathname.name) task_pathname = Path(self.i_context.task_filename) task_doc = await self.repo.docs.task.get(filename=task_pathname.name) - src_file_repo = self.repo.with_src_path(self.context.src_workspace).srcs code_blocks = [] for filename in self.i_context.codes_filenames: - code_doc = await src_file_repo.get(filename) - code_block = f"```python\n{code_doc.content}\n```\n-----" + code_doc = await self.repo.srcs.get(filename) + code_block = f"```{get_markdown_code_block_type(filename)}\n{code_doc.content}\n```\n---\n" code_blocks.append(code_block) format_example = FORMAT_EXAMPLE prompt = PROMPT_TEMPLATE.format( diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 67b859d23..7f225d469 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -16,17 +16,18 @@ """ import json +from pathlib import Path +from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE -from metagpt.const import BUGFIX_FILENAME, REQUIREMENT_FILENAME from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult -from metagpt.utils.common import CodeParser +from metagpt.utils.common import CodeParser, get_markdown_code_block_type from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import EditorReporter @@ -44,9 +45,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc {task} ## Legacy Code -```Code {code} -``` ## Debug logs ```text @@ -61,14 +60,14 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc ``` # Format example -## Code: {filename} +## Code: {demo_filename}.py ```python -## {filename} +## {demo_filename}.py ... ``` -## Code: {filename} +## Code: {demo_filename}.js ```javascript -// {filename} +// {demo_filename}.js ... ``` @@ -89,6 +88,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): name: str = "WriteCode" i_context: Document = Field(default_factory=Document) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def write_code(self, prompt) -> str: @@ -97,10 +98,16 @@ class WriteCode(Action): return code async def run(self, *args, **kwargs) -> CodingContext: - bug_feedback = await self.repo.docs.get(filename=BUGFIX_FILENAME) + bug_feedback = None + if self.input_args and hasattr(self.input_args, "issue_filename"): + bug_feedback = await Document.load(self.input_args.issue_filename) coding_context = CodingContext.loads(self.i_context.content) + if not coding_context.code_plan_and_change_doc: + coding_context.code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get( + filename=coding_context.task_doc.filename + ) test_doc = await self.repo.test_outputs.get(filename="test_" + coding_context.filename + ".json") - requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME) + requirement_doc = await Document.load(self.input_args.requirements_filename) summary_doc = None if coding_context.design_doc and coding_context.design_doc.filename: summary_doc = await self.repo.docs.code_summary.get(filename=coding_context.design_doc.filename) @@ -109,29 +116,28 @@ class WriteCode(Action): test_detail = RunCodeResult.loads(test_doc.content) logs = test_detail.stderr - if bug_feedback: - code_context = coding_context.code_doc.content - elif self.config.inc: + if self.config.inc or bug_feedback: code_context = await self.get_codes( coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo, use_inc=True ) else: code_context = await self.get_codes( - coding_context.task_doc, - exclude=self.i_context.filename, - project_repo=self.repo.with_src_path(self.context.src_workspace), + coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo ) if self.config.inc: prompt = REFINED_TEMPLATE.format( user_requirement=requirement_doc.content if requirement_doc else "", - code_plan_and_change=str(coding_context.code_plan_and_change_doc), + code_plan_and_change=coding_context.code_plan_and_change_doc.content + if coding_context.code_plan_and_change_doc + else "", design=coding_context.design_doc.content if coding_context.design_doc else "", task=coding_context.task_doc.content if coding_context.task_doc else "", code=code_context, logs=logs, feedback=bug_feedback.content if bug_feedback else "", filename=self.i_context.filename, + demo_filename=Path(self.i_context.filename).stem, summary_log=summary_doc.content if summary_doc else "", ) else: @@ -142,6 +148,7 @@ class WriteCode(Action): logs=logs, feedback=bug_feedback.content if bug_feedback else "", filename=self.i_context.filename, + demo_filename=Path(self.i_context.filename).stem, summary_log=summary_doc.content if summary_doc else "", ) logger.info(f"Writing {coding_context.filename}..") @@ -150,8 +157,9 @@ class WriteCode(Action): code = await self.write_code(prompt) if not coding_context.code_doc: # avoid root_path pydantic ValidationError if use WriteCode alone - root_path = self.context.src_workspace if self.context.src_workspace else "" - coding_context.code_doc = Document(filename=coding_context.filename, root_path=str(root_path)) + coding_context.code_doc = Document( + filename=coding_context.filename, root_path=str(self.repo.src_relative_path) + ) coding_context.code_doc.content = code await reporter.async_report(self.repo.workdir / coding_context.code_doc.root_relative_path, "path") return coding_context @@ -178,35 +186,32 @@ class WriteCode(Action): code_filenames = m.get(TASK_LIST.key, []) if not use_inc else m.get(REFINED_TASK_LIST.key, []) codes = [] src_file_repo = project_repo.srcs - # Incremental development scenario if use_inc: - src_files = src_file_repo.all_files - # Get the old workspace contained the old codes and old workspace are created in previous CodePlanAndChange - old_file_repo = project_repo.git_repo.new_file_repository(relative_path=project_repo.old_workspace) - old_files = old_file_repo.all_files - # Get the union of the files in the src and old workspaces - union_files_list = list(set(src_files) | set(old_files)) - for filename in union_files_list: + for filename in src_file_repo.all_files: + code_block_type = get_markdown_code_block_type(filename) # Exclude the current file from the all code snippets if filename == exclude: # If the file is in the old workspace, use the old code # Exclude unnecessary code to maintain a clean and focused main.py file, ensuring only relevant and # essential functionality is included for the project’s requirements - if filename in old_files and filename != "main.py": + if filename != "main.py": # Use old code - doc = await old_file_repo.get(filename=filename) + doc = await src_file_repo.get(filename=filename) # If the file is in the src workspace, skip it else: continue - codes.insert(0, f"-----Now, {filename} to be rewritten\n```{doc.content}```\n=====") + codes.insert( + 0, f"### The name of file to rewrite: `{filename}`\n```{code_block_type}\n{doc.content}```\n" + ) + logger.info(f"Prepare to rewrite `{filename}`") # The code snippets are generated from the src workspace else: doc = await src_file_repo.get(filename=filename) # If the file does not exist in the src workspace, skip it if not doc: continue - codes.append(f"----- {filename}\n```{doc.content}```") + codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n") # Normal scenario else: @@ -217,6 +222,7 @@ class WriteCode(Action): doc = await src_file_repo.get(filename=filename) if not doc: continue - codes.append(f"----- {filename}\n```{doc.content}```") + code_block_type = get_markdown_code_block_type(filename) + codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n") return "\n".join(codes) diff --git a/metagpt/actions/write_code_plan_and_change_an.py b/metagpt/actions/write_code_plan_and_change_an.py index a3c0e50a4..31482a94d 100644 --- a/metagpt/actions/write_code_plan_and_change_an.py +++ b/metagpt/actions/write_code_plan_and_change_an.py @@ -5,15 +5,16 @@ @Author : mannaandpoem @File : write_code_plan_and_change_an.py """ -import os -from typing import List +from typing import List, Optional -from pydantic import Field +from pydantic import BaseModel, Field from metagpt.actions.action import Action from metagpt.actions.action_node import ActionNode from metagpt.logs import logger -from metagpt.schema import CodePlanAndChangeContext +from metagpt.schema import CodePlanAndChangeContext, Document +from metagpt.utils.common import get_markdown_code_block_type +from metagpt.utils.project_repo import ProjectRepo DEVELOPMENT_PLAN = ActionNode( key="Development Plan", @@ -162,9 +163,8 @@ Role: You are a professional engineer; The main goal is to complete incremental {task} ## Legacy Code -```Code {code} -``` + ## Debug logs ```text @@ -179,14 +179,14 @@ Role: You are a professional engineer; The main goal is to complete incremental ``` # Format example -## Code: {filename} +## Code: {demo_filename}.py ```python -## {filename} +## {demo_filename}.py ... ``` -## Code: {filename} +## Code: {demo_filename}.js ```javascript -// {filename} +// {demo_filename}.js ... ``` @@ -211,13 +211,15 @@ WRITE_CODE_PLAN_AND_CHANGE_NODE = ActionNode.from_children("WriteCodePlanAndChan class WriteCodePlanAndChange(Action): name: str = "WriteCodePlanAndChange" i_context: CodePlanAndChangeContext = Field(default_factory=CodePlanAndChangeContext) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) async def run(self, *args, **kwargs): self.llm.system_prompt = "You are a professional software engineer, your primary responsibility is to " "meticulously craft comprehensive incremental development plan and deliver detailed incremental change" - prd_doc = await self.repo.docs.prd.get(filename=self.i_context.prd_filename) - design_doc = await self.repo.docs.system_design.get(filename=self.i_context.design_filename) - task_doc = await self.repo.docs.task.get(filename=self.i_context.task_filename) + prd_doc = await Document.load(filename=self.i_context.prd_filename) + design_doc = await Document.load(filename=self.i_context.design_filename) + task_doc = await Document.load(filename=self.i_context.task_filename) context = CODE_PLAN_AND_CHANGE_CONTEXT.format( requirement=f"```text\n{self.i_context.requirement}\n```", issue=f"```text\n{self.i_context.issue}\n```", @@ -230,8 +232,9 @@ class WriteCodePlanAndChange(Action): return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(context=context, llm=self.llm, schema="json") async def get_old_codes(self) -> str: - self.repo.old_workspace = self.repo.git_repo.workdir / os.path.basename(self.config.project_path) - old_file_repo = self.repo.git_repo.new_file_repository(relative_path=self.repo.old_workspace) - old_codes = await old_file_repo.get_all() - codes = [f"----- {code.filename}\n```{code.content}```" for code in old_codes] + old_codes = await self.repo.srcs.get_all() + codes = [ + f"### File Name: `{code.filename}`\n```{get_markdown_code_block_type(code.filename)}\n{code.content}```\n" + for code in old_codes + ] return "\n".join(codes) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index f0faea701..3912095df 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -7,16 +7,17 @@ @Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the WriteCode object, rather than passing them in when calling the run function. """ +from typing import Optional -from pydantic import Field +from pydantic import BaseModel, Field from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions import WriteCode from metagpt.actions.action import Action -from metagpt.const import REQUIREMENT_FILENAME from metagpt.logs import logger -from metagpt.schema import CodingContext +from metagpt.schema import CodingContext, Document from metagpt.utils.common import CodeParser +from metagpt.utils.project_repo import ProjectRepo PROMPT_TEMPLATE = """ # System @@ -126,6 +127,8 @@ or class WriteCodeReview(Action): name: str = "WriteCodeReview" i_context: CodingContext = Field(default_factory=CodingContext) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6)) async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename): @@ -150,7 +153,7 @@ class WriteCodeReview(Action): code_context = await WriteCode.get_codes( self.i_context.task_doc, exclude=self.i_context.filename, - project_repo=self.repo.with_src_path(self.context.src_workspace), + project_repo=self.repo, use_inc=self.config.inc, ) @@ -160,7 +163,7 @@ class WriteCodeReview(Action): "## Code Files\n" + code_context + "\n", ] if self.config.inc: - requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME) + requirement_doc = await Document.load(filename=self.input_args.requirements_filename) insert_ctx_list = [ "## User New Requirements\n" + str(requirement_doc) + "\n", "## Code Plan And Change\n" + str(self.i_context.code_plan_and_change_doc) + "\n", diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index a4f6e1dd1..3275619f7 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -15,6 +15,9 @@ from __future__ import annotations import json from pathlib import Path +from typing import Optional + +from pydantic import BaseModel, Field from metagpt.actions import Action, ActionOutput from metagpt.actions.action_node import ActionNode @@ -37,6 +40,7 @@ from metagpt.schema import AIMessage, Document, Documents, Message from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.mermaid import mermaid_to_file +from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import DocsReporter, GalleryReporter CONTEXT_TEMPLATE = """ @@ -66,10 +70,30 @@ class WritePRD(Action): 3. Requirement update: If the requirement is an update, the PRD document will be updated. """ + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) + async def run(self, with_messages, *args, **kwargs) -> Message: """Run the action.""" - req: Document = await self.repo.requirement - docs: list[Document] = await self.repo.docs.prd.get_all() + self.input_args = with_messages[-1].instruct_content + if not self.input_args: + self.repo = ProjectRepo(self.config.project_path) + await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[-1].content) + self.input_args = AIMessage.create_instruct_value( + kvs={ + "project_path": self.config.project_path, + "requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME), + "prd_filenames": [str(self.repo.docs.prd.workdir / i) for i in self.repo.docs.prd.all_files], + }, + class_name="PrepareDocumentsOutput", + ) + else: + self.repo = ProjectRepo(self.input_args.project_path) + req = await Document.load(filename=self.input_args.requirements_filename) + docs: list[Document] = [ + await Document.load(filename=i, project_path=self.repo.workdir) for i in self.input_args.prd_filenames + ] + if not req: raise FileNotFoundError("No requirement document found.") @@ -82,10 +106,14 @@ class WritePRD(Action): # if requirement is related to other documents, update them, otherwise create a new one if related_docs := await self.get_related_docs(req, docs): logger.info(f"Requirement update detected: {req.content}") - await self._handle_requirement_update(req, related_docs) + await self._handle_requirement_update(req=req, related_docs=related_docs) else: logger.info(f"New requirement detected: {req.content}") await self._handle_new_requirement(req) + kvs = self.input_args.model_dump() + kvs["changed_prd_filenames"] = [ + str(self.repo.docs.prd.workdir / i) for i in list(self.repo.docs.prd.changed_files.keys()) + ] return AIMessage( content="PRD is completed. " + "\n".join( @@ -93,6 +121,7 @@ class WritePRD(Action): + list(self.repo.resources.prd.changed_files.keys()) + list(self.repo.resources.competitive_analysis.changed_files.keys()) ), + instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WritePRDOutput"), cause_by=self, ) @@ -103,6 +132,14 @@ class WritePRD(Action): return AIMessage( content=f"A new issue is received: {BUGFIX_FILENAME}", cause_by=FixBug, + instruct_content=AIMessage.create_instruct_value( + { + "project_path": str(self.repo.workdir), + "issue_filename": str(self.repo.docs.workdir / BUGFIX_FILENAME), + "requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME), + }, + class_name="IssueDetail", + ), send_to="Alex", # the name of Engineer ) @@ -128,7 +165,7 @@ class WritePRD(Action): async def _handle_requirement_update(self, req: Document, related_docs: list[Document]) -> ActionOutput: # ... requirement update logic ... for doc in related_docs: - await self._update_prd(req, doc) + await self._update_prd(req=req, prd_doc=doc) return Documents.from_iterable(documents=related_docs).to_action_output() async def _is_bugfix(self, context: str) -> bool: @@ -159,7 +196,7 @@ class WritePRD(Action): async def _update_prd(self, req: Document, prd_doc: Document) -> Document: async with DocsReporter(enable_llm_stream=True) as reporter: await reporter.async_report({"type": "prd"}, "meta") - new_prd_doc: Document = await self._merge(req, prd_doc) + new_prd_doc: Document = await self._merge(req=req, related_doc=prd_doc) await self.repo.docs.prd.save_doc(doc=new_prd_doc) await self._save_competitive_analysis(new_prd_doc) md = await self.repo.resources.prd.save_pdf(doc=new_prd_doc) diff --git a/metagpt/actions/write_prd_an.py b/metagpt/actions/write_prd_an.py index a33685cd3..1ceb2aade 100644 --- a/metagpt/actions/write_prd_an.py +++ b/metagpt/actions/write_prd_an.py @@ -5,7 +5,7 @@ @Author : alexanderwu @File : write_prd_an.py """ -from typing import List +from typing import List, Union from metagpt.actions.action_node import ActionNode @@ -132,7 +132,7 @@ REQUIREMENT_ANALYSIS = ActionNode( REFINED_REQUIREMENT_ANALYSIS = ActionNode( key="Refined Requirement Analysis", - expected_type=List[str], + expected_type=Union[List[str], str], instruction="Review and refine the existing requirement analysis into a string list to align with the evolving needs of the project " "due to incremental development. Ensure the analysis comprehensively covers the new features and enhancements " "required for the refined project scope.", diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d9e375a9a..111e534a6 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -24,20 +24,15 @@ from collections import defaultdict from pathlib import Path from typing import List, Optional, Set -from metagpt.actions import ( - Action, - UserRequirement, - WriteCode, - WriteCodeReview, - WriteTasks, -) +from pydantic import BaseModel, Field + +from metagpt.actions import UserRequirement, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.fix_bug import FixBug from metagpt.actions.prepare_documents import PrepareDocuments from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST from metagpt.actions.summarize_code import SummarizeCode from metagpt.actions.write_code_plan_and_change_an import WriteCodePlanAndChange from metagpt.const import ( - BUGFIX_FILENAME, CODE_PLAN_AND_CHANGE_FILE_REPO, MESSAGE_ROUTE_TO_SELF, REQUIREMENT_FILENAME, @@ -63,6 +58,7 @@ from metagpt.utils.common import ( init_python_folder, ) from metagpt.utils.git_repository import ChangeType +from metagpt.utils.project_repo import ProjectRepo IS_PASS_PROMPT = """ {context} @@ -100,6 +96,8 @@ class Engineer(Role): summarize_todos: list = [] next_todo_action: str = "" n_summarize: int = 0 + input_args: Optional[BaseModel] = Field(default=None, exclude=True) + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -139,14 +137,20 @@ class Engineer(Role): coding_context = await todo.run() # Code review if review: - action = WriteCodeReview(i_context=coding_context, context=self.context, llm=self.llm) + action = WriteCodeReview( + i_context=coding_context, + repo=self.repo, + input_args=self.input_args, + context=self.context, + llm=self.llm, + ) self._init_action(action) coding_context = await action.run() dependencies = {coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path} if self.config.inc: dependencies.add(coding_context.code_plan_and_change_doc.root_relative_path) - await self.project_repo.srcs.save( + await self.repo.srcs.save( filename=coding_context.filename, dependencies=list(dependencies), content=coding_context.code_doc.content, @@ -186,9 +190,9 @@ class Engineer(Role): summary_filename = Path(todo.i_context.design_filename).with_suffix(".md").name dependencies = {todo.i_context.design_filename, todo.i_context.task_filename} for filename in todo.i_context.codes_filenames: - rpath = self.project_repo.src_relative_path / filename + rpath = self.repo.src_relative_path / filename dependencies.add(str(rpath)) - await self.project_repo.resources.code_summary.save( + await self.repo.resources.code_summary.save( filename=summary_filename, content=summary, dependencies=dependencies ) is_pass, reason = await self._is_pass(summary) @@ -196,23 +200,39 @@ class Engineer(Role): todo.i_context.reason = reason tasks.append(todo.i_context.model_dump()) - await self.project_repo.docs.code_summary.save( + await self.repo.docs.code_summary.save( filename=Path(todo.i_context.design_filename).name, content=todo.i_context.model_dump_json(), dependencies=dependencies, ) else: - await self.project_repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name) + await self.repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name) self.summarize_todos = [] logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}") if not tasks or self.config.max_auto_summarize_code == 0: self.n_summarize = 0 + kvs = self.input_args.model_dump() + kvs["changed_src_filenames"] = [ + str(self.repo.srcs.workdir / i) for i in list(self.repo.srcs.changed_files.keys()) + ] + if self.repo.docs.code_plan_and_change.changed_files: + kvs["changed_code_plan_and_change_filenames"] = [ + str(self.repo.docs.code_plan_and_change.workdir / i) + for i in list(self.repo.docs.code_plan_and_change.changed_files.keys()) + ] + if self.repo.docs.code_summary.changed_files: + kvs["changed_code_summary_filenames"] = [ + str(self.repo.docs.code_summary.workdir / i) + for i in list(self.repo.docs.code_summary.changed_files.keys()) + ] return AIMessage( - content=f"Coding is complete. The source code is at {self.project_repo.workdir.name}/{self.project_repo.srcs.root_path}, containing: " + content=f"Coding is complete. The source code is at {self.repo.workdir.name}/{self.repo.srcs.root_path}, containing: " + "\n".join( - list(self.project_repo.resources.code_summary.changed_files.keys()) - + list(self.project_repo.srcs.changed_files.keys()) + list(self.repo.resources.code_summary.changed_files.keys()) + + list(self.repo.srcs.changed_files.keys()) + + list(self.repo.resources.code_plan_and_change.changed_files.keys()) ), + instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="SummarizeCodeOutput"), cause_by=SummarizeCode, send_to="Edward", # The name of QaEngineer ) @@ -227,15 +247,15 @@ class Engineer(Role): code_plan_and_change = node.instruct_content.model_dump_json() dependencies = { REQUIREMENT_FILENAME, - str(self.project_repo.docs.prd.root_path / self.rc.todo.i_context.prd_filename), - str(self.project_repo.docs.system_design.root_path / self.rc.todo.i_context.design_filename), - str(self.project_repo.docs.task.root_path / self.rc.todo.i_context.task_filename), + str(Path(self.rc.todo.i_context.prd_filename).relative_to(self.repo.workdir)), + str(Path(self.rc.todo.i_context.design_filename).relative_to(self.repo.workdir)), + str(Path(self.rc.todo.i_context.task_filename).relative_to(self.repo.workdir)), } code_plan_and_change_filepath = Path(self.rc.todo.i_context.design_filename) - await self.project_repo.docs.code_plan_and_change.save( + await self.repo.docs.code_plan_and_change.save( filename=code_plan_and_change_filepath.name, content=code_plan_and_change, dependencies=dependencies ) - await self.project_repo.resources.code_plan_and_change.save( + await self.repo.resources.code_plan_and_change.save( filename=code_plan_and_change_filepath.with_suffix(".md").name, content=node.content, dependencies=dependencies, @@ -250,10 +270,11 @@ class Engineer(Role): return True, rsp return False, rsp - async def _think(self) -> Action | None: + async def _think(self) -> bool: if not self.rc.news: - return None + return False msg = self.rc.news[0] + input_args = msg.instruct_content if msg.cause_by == any_to_str(UserRequirement): self.rc.todo = PrepareDocuments( key_descriptions={ @@ -263,42 +284,47 @@ class Engineer(Role): context=self.context, send_to=any_to_str(self), ) - return self.rc.todo - - if not self.src_workspace: - self.src_workspace = get_project_srcs_path(self.project_repo.workdir) + self.repo = ProjectRepo(input_args.project_path) + self.input_args = input_args + return bool(self.rc.todo) + elif msg.cause_by in {any_to_str(WriteTasks), any_to_str(FixBug)}: + self.input_args = input_args + self.repo = ProjectRepo(input_args.project_path) + if self.repo.src_relative_path is None: + path = get_project_srcs_path(self.repo.workdir) + self.repo.with_src_path(path) write_plan_and_change_filters = any_to_str_set([PrepareDocuments, WriteTasks, FixBug]) write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode]) summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if self.config.inc and msg.cause_by in write_plan_and_change_filters: logger.debug(f"TODO WriteCodePlanAndChange:{msg.model_dump_json()}") await self._new_code_plan_and_change_action(cause_by=msg.cause_by) - return self.rc.todo + return bool(self.rc.todo) if msg.cause_by in write_code_filters: logger.debug(f"TODO WriteCode:{msg.model_dump_json()}") await self._new_code_actions() - return self.rc.todo + return bool(self.rc.todo) if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): logger.debug(f"TODO SummarizeCode:{msg.model_dump_json()}") await self._new_summarize_actions() - return self.rc.todo - return None + return bool(self.rc.todo) + return False async def _new_coding_context(self, filename, dependency) -> Optional[CodingContext]: - old_code_doc = await self.project_repo.srcs.get(filename) + old_code_doc = await self.repo.srcs.get(filename) if not old_code_doc: - old_code_doc = Document(root_path=str(self.project_repo.src_relative_path), filename=filename, content="") + old_code_doc = Document(root_path=str(self.repo.src_relative_path), filename=filename, content="") dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)} task_doc = None design_doc = None code_plan_and_change_doc = await self._get_any_code_plan_and_change() if await self._is_fixbug() else None for i in dependencies: if str(i.parent) == TASK_FILE_REPO: - task_doc = await self.project_repo.docs.task.get(i.name) + task_doc = await self.repo.docs.task.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: - design_doc = await self.project_repo.docs.system_design.get(i.name) + design_doc = await self.repo.docs.system_design.get(i.name) elif str(i.parent) == CODE_PLAN_AND_CHANGE_FILE_REPO: - code_plan_and_change_doc = await self.project_repo.docs.code_plan_and_change.get(i.name) + code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(i.name) if not task_doc or not design_doc: if filename == "__init__.py": # `__init__.py` created by `init_python_folder` return None @@ -318,34 +344,66 @@ class Engineer(Role): if not context: return None # `__init__.py` created by `init_python_folder` coding_doc = Document( - root_path=str(self.project_repo.src_relative_path), filename=filename, content=context.model_dump_json() + root_path=str(self.repo.src_relative_path), filename=filename, content=context.model_dump_json() ) return coding_doc async def _new_code_actions(self): bug_fix = await self._is_fixbug() # Prepare file repos - changed_src_files = self.project_repo.srcs.changed_files + changed_src_files = self.repo.srcs.changed_files if self.context.kwargs.src_filename: changed_src_files = {self.context.kwargs.src_filename: ChangeType.UNTRACTED} if bug_fix: - changed_src_files = self.project_repo.srcs.all_files - changed_task_files = self.project_repo.docs.task.changed_files + changed_src_files = self.repo.srcs.all_files changed_files = Documents() # Recode caused by upstream changes. - for filename in changed_task_files: - design_doc = await self.project_repo.docs.system_design.get(filename) - task_doc = await self.project_repo.docs.task.get(filename) - code_plan_and_change_doc = await self.project_repo.docs.code_plan_and_change.get(filename) + if hasattr(self.input_args, "changed_task_filenames"): + changed_task_filenames = self.input_args.changed_task_filenames + else: + changed_task_filenames = [ + str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys()) + ] + for filename in changed_task_filenames: + task_filename = Path(filename) + design_filename = None + if hasattr(self.input_args, "changed_system_design_filenames"): + changed_system_design_filenames = self.input_args.changed_system_design_filenames + else: + changed_system_design_filenames = [ + str(self.repo.docs.system_design.workdir / i) + for i in list(self.repo.docs.system_design.changed_files.keys()) + ] + for i in changed_system_design_filenames: + if task_filename.name == Path(i).name: + design_filename = Path(i) + break + code_plan_and_change_filename = None + if hasattr(self.input_args, "changed_code_plan_and_change_filenames"): + changed_code_plan_and_change_filenames = self.input_args.changed_code_plan_and_change_filenames + else: + changed_code_plan_and_change_filenames = [ + str(self.repo.docs.code_plan_and_change.workdir / i) + for i in list(self.repo.docs.code_plan_and_change.changed_files.keys()) + ] + for i in changed_code_plan_and_change_filenames: + if task_filename.name == Path(i).name: + code_plan_and_change_filename = Path(i) + break + design_doc = await Document.load(filename=design_filename, project_path=self.repo.workdir) + task_doc = await Document.load(filename=task_filename, project_path=self.repo.workdir) + code_plan_and_change_doc = await Document.load( + filename=code_plan_and_change_filename, project_path=self.repo.workdir + ) task_list = self._parse_tasks(task_doc) await self._init_python_folder(task_list) for task_filename in task_list: if self.context.kwargs.src_filename and task_filename != self.context.kwargs.src_filename: continue - old_code_doc = await self.project_repo.srcs.get(task_filename) + old_code_doc = await self.repo.srcs.get(task_filename) if not old_code_doc: old_code_doc = Document( - root_path=str(self.project_repo.src_relative_path), filename=task_filename, content="" + root_path=str(self.repo.src_relative_path), filename=task_filename, content="" ) if not code_plan_and_change_doc: context = CodingContext( @@ -360,7 +418,7 @@ class Engineer(Role): code_plan_and_change_doc=code_plan_and_change_doc, ) coding_doc = Document( - root_path=str(self.project_repo.src_relative_path), + root_path=str(self.repo.src_relative_path), filename=task_filename, content=context.model_dump_json(), ) @@ -371,10 +429,11 @@ class Engineer(Role): ) changed_files.docs[task_filename] = coding_doc self.code_todos = [ - WriteCode(i_context=i, context=self.context, llm=self.llm) for i in changed_files.docs.values() + WriteCode(i_context=i, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm) + for i in changed_files.docs.values() ] # Code directly modified by the user. - dependency = await self.git_repo.get_dependency() + dependency = await self.repo.git_repo.get_dependency() for filename in changed_src_files: if filename in changed_files.docs: continue @@ -382,24 +441,30 @@ class Engineer(Role): if not coding_doc: continue # `__init__.py` created by `init_python_folder` changed_files.docs[filename] = coding_doc - self.code_todos.append(WriteCode(i_context=coding_doc, context=self.context, llm=self.llm)) + self.code_todos.append( + WriteCode( + i_context=coding_doc, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm + ) + ) if self.code_todos: self.set_todo(self.code_todos[0]) async def _new_summarize_actions(self): - src_files = self.project_repo.srcs.all_files + src_files = self.repo.srcs.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). summarizations = defaultdict(list) for filename in src_files: - dependencies = await self.project_repo.srcs.get_dependency(filename=filename) + dependencies = await self.repo.srcs.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=list(dependencies)) summarizations[ctx].append(filename) for ctx, filenames in summarizations.items(): if not ctx.design_filename or not ctx.task_filename: continue # cause by `__init__.py` which is created by `init_python_folder` ctx.codes_filenames = filenames - new_summarize = SummarizeCode(i_context=ctx, context=self.context, llm=self.llm) + new_summarize = SummarizeCode( + i_context=ctx, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm + ) for i, act in enumerate(self.summarize_todos): if act.i_context.task_filename == new_summarize.i_context.task_filename: self.summarize_todos[i] = new_summarize @@ -412,16 +477,37 @@ class Engineer(Role): async def _new_code_plan_and_change_action(self, cause_by: str): """Create a WriteCodePlanAndChange action for subsequent to-do actions.""" - files = self.project_repo.all_files options = {} if cause_by != any_to_str(FixBug): - requirement_doc = await self.project_repo.docs.get(REQUIREMENT_FILENAME) + requirement_doc = await Document.load(filename=self.input_args.requirements_filename) options["requirement"] = requirement_doc.content else: - fixbug_doc = await self.project_repo.docs.get(BUGFIX_FILENAME) + fixbug_doc = await Document.load(filename=self.input_args.issue_filename) options["issue"] = fixbug_doc.content - code_plan_and_change_ctx = CodePlanAndChangeContext.loads(files, **options) - self.rc.todo = WriteCodePlanAndChange(i_context=code_plan_and_change_ctx, context=self.context, llm=self.llm) + # The code here is flawed: if there are multiple unrelated requirements, this piece of logic will break + if hasattr(self.input_args, "changed_prd_filenames"): + code_plan_and_change_ctx = CodePlanAndChangeContext( + requirement=options.get("requirement", ""), + issue=options.get("issue", ""), + prd_filename=self.input_args.changed_prd_filenames[0], + design_filename=self.input_args.changed_system_design_filenames[0], + task_filename=self.input_args.changed_task_filenames[0], + ) + else: + code_plan_and_change_ctx = CodePlanAndChangeContext( + requirement=options.get("requirement", ""), + issue=options.get("issue", ""), + prd_filename=str(self.repo.docs.prd.workdir / self.repo.docs.prd.all_files[0]), + design_filename=str(self.repo.docs.system_design.workdir / self.repo.docs.system_design.all_files[0]), + task_filename=str(self.repo.docs.task.workdir / self.repo.docs.task.all_files[0]), + ) + self.rc.todo = WriteCodePlanAndChange( + i_context=code_plan_and_change_ctx, + repo=self.repo, + input_args=self.input_args, + context=self.context, + llm=self.llm, + ) @property def action_description(self) -> str: @@ -433,17 +519,16 @@ class Engineer(Role): filename = Path(i) if filename.suffix != ".py": continue - workdir = self.src_workspace / filename.parent + workdir = self.repo.srcs.workdir / filename.parent await init_python_folder(workdir) async def _is_fixbug(self) -> bool: - fixbug_doc = await self.project_repo.docs.get(BUGFIX_FILENAME) - return bool(fixbug_doc and fixbug_doc.content) + return bool(self.input_args and hasattr(self.input_args, "issue_filename")) async def _get_any_code_plan_and_change(self) -> Optional[Document]: - changed_files = self.project_repo.docs.code_plan_and_change.changed_files + changed_files = self.repo.docs.code_plan_and_change.changed_files for filename in changed_files.keys(): - doc = await self.project_repo.docs.code_plan_and_change.get(filename) + doc = await self.repo.docs.code_plan_and_change.get(filename) if doc and doc.content: return doc return None diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 4beab5366..58d8076ab 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -7,10 +7,12 @@ @Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135. """ + from metagpt.actions import UserRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments -from metagpt.roles.role import Role, RoleReactMode +from metagpt.roles.role import Role from metagpt.utils.common import any_to_name, any_to_str +from metagpt.utils.git_repository import GitRepository class ProductManager(Role): @@ -35,12 +37,11 @@ class ProductManager(Role): self.enable_memory = False self.set_actions([PrepareDocuments(send_to=any_to_str(self)), WritePRD]) self._watch([UserRequirement, PrepareDocuments]) - self.rc.react_mode = RoleReactMode.BY_ORDER self.todo_action = any_to_name(WritePRD) async def _think(self) -> bool: """Decide what to do""" - if self.git_repo and not self.config.git_reinit: + if GitRepository.is_git_dir(self.config.project_path) and not self.config.git_reinit: self._set_state(1) else: self._set_state(0) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index f76baff3f..48ed24c2c 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -14,6 +14,9 @@ @Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results of SummarizeCode. """ +from typing import Optional + +from pydantic import BaseModel, Field from metagpt.actions import DebugError, RunCode, UserRequirement, WriteTest from metagpt.actions.prepare_documents import PrepareDocuments @@ -25,9 +28,11 @@ from metagpt.schema import AIMessage, Document, Message, RunCodeContext, Testing from metagpt.utils.common import ( any_to_str, any_to_str_set, + get_project_srcs_path, init_python_folder, parse_recipient, ) +from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.report import EditorReporter @@ -41,6 +46,8 @@ class QaEngineer(Role): ) test_round_allowed: int = 5 test_round: int = 0 + repo: Optional[ProjectRepo] = Field(default=None, exclude=True) + input_args: Optional[BaseModel] = Field(default=None, exclude=True) def __init__(self, **kwargs): super().__init__(**kwargs) @@ -57,22 +64,21 @@ class QaEngineer(Role): self.test_round = 0 async def _write_test(self, message: Message) -> None: - src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs reqa_file = self.context.kwargs.reqa_file or self.config.reqa_file - changed_files = {reqa_file} if reqa_file else set(src_file_repo.changed_files.keys()) + changed_files = {reqa_file} if reqa_file else set(self.repo.srcs.changed_files.keys()) for filename in changed_files: # write tests if not filename or "test" in filename: continue - code_doc = await src_file_repo.get(filename) - if not code_doc: + code_doc = await self.repo.srcs.get(filename) + if not code_doc or not code_doc.content: continue if not code_doc.filename.endswith(".py"): continue - test_doc = await self.project_repo.tests.get("test_" + code_doc.filename) + test_doc = await self.repo.tests.get("test_" + code_doc.filename) if not test_doc: test_doc = Document( - root_path=str(self.project_repo.tests.root_path), filename="test_" + code_doc.filename, content="" + root_path=str(self.repo.tests.root_path), filename="test_" + code_doc.filename, content="" ) logger.info(f"Writing {test_doc.filename}..") context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc) @@ -81,40 +87,38 @@ class QaEngineer(Role): async with EditorReporter(enable_llm_stream=True) as reporter: await reporter.async_report({"type": "test", "filename": test_doc.filename}, "meta") - doc = await self.project_repo.tests.save_doc( + doc = await self.repo.tests.save_doc( doc=context.test_doc, dependencies={context.code_doc.root_relative_path} ) - await reporter.async_report(self.project_repo.workdir / doc.root_relative_path, "path") + await reporter.async_report(self.repo.workdir / doc.root_relative_path, "path") # prepare context for run tests in next round run_code_context = RunCodeContext( command=["python", context.test_doc.root_relative_path], code_filename=context.code_doc.filename, test_filename=context.test_doc.filename, - working_directory=str(self.project_repo.workdir), + working_directory=str(self.repo.workdir), additional_python_paths=[str(self.context.src_workspace)], ) self.publish_message( AIMessage(content=run_code_context.model_dump_json(), cause_by=WriteTest, send_to=MESSAGE_ROUTE_TO_SELF) ) - logger.info(f"Done {str(self.project_repo.tests.workdir)} generating.") + logger.info(f"Done {str(self.repo.tests.workdir)} generating.") async def _run_code(self, msg): run_code_context = RunCodeContext.loads(msg.content) - src_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get( - run_code_context.code_filename - ) + src_doc = await self.repo.srcs.get(run_code_context.code_filename) if not src_doc: return - test_doc = await self.project_repo.tests.get(run_code_context.test_filename) + test_doc = await self.repo.tests.get(run_code_context.test_filename) if not test_doc: return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content result = await RunCode(i_context=run_code_context, context=self.context, llm=self.llm).run() run_code_context.output_filename = run_code_context.test_filename + ".json" - await self.project_repo.test_outputs.save( + await self.repo.test_outputs.save( filename=run_code_context.output_filename, content=result.model_dump_json(), dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, @@ -124,31 +128,53 @@ class QaEngineer(Role): # the recipient might be Engineer or myself recipient = parse_recipient(result.summary) mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} - self.publish_message( - AIMessage( - content=run_code_context.model_dump_json(), - cause_by=RunCode, - send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), + if recipient != "Engineer": + self.publish_message( + AIMessage( + content=run_code_context.model_dump_json(), + cause_by=RunCode, + instruct_content=self.input_args, + send_to=MESSAGE_ROUTE_TO_SELF, + ) + ) + else: + kvs = self.input_args.model_dump() + kvs["changed_test_filenames"] = [ + str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys()) + ] + self.publish_message( + AIMessage( + content=run_code_context.model_dump_json(), + cause_by=RunCode, + instruct_content=self.input_args, + send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE), + ) ) - ) async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) - code = await DebugError(i_context=run_code_context, context=self.context, llm=self.llm).run() - await self.project_repo.tests.save(filename=run_code_context.test_filename, content=code) + code = await DebugError( + i_context=run_code_context, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm + ).run() + await self.repo.tests.save(filename=run_code_context.test_filename, content=code) run_code_context.output = None self.publish_message( AIMessage(content=run_code_context.model_dump_json(), cause_by=DebugError, send_to=MESSAGE_ROUTE_TO_SELF) ) async def _act(self) -> Message: - if self.project_path: - await init_python_folder(self.project_repo.tests.workdir) + if self.input_args.project_path: + await init_python_folder(self.repo.tests.workdir) if self.test_round > self.test_round_allowed: + kvs = self.input_args.model_dump() + kvs["changed_test_filenames"] = [ + str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys()) + ] result_msg = AIMessage( content=f"Exceeding {self.test_round_allowed} rounds of tests, stop. " - + "\n".join(list(self.project_repo.tests.changed_files.keys())), + + "\n".join(list(self.repo.tests.changed_files.keys())), cause_by=WriteTest, + instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTestOutput"), send_to=MESSAGE_ROUTE_TO_NONE, ) return result_msg @@ -171,8 +197,13 @@ class QaEngineer(Role): elif msg.cause_by == any_to_str(UserRequirement): return await self._parse_user_requirement(msg) self.test_round += 1 + kvs = self.input_args.model_dump() + kvs["changed_test_filenames"] = [ + str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys()) + ] return AIMessage( content=f"Round {self.test_round} of tests done", + instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTestOutput"), cause_by=WriteTest, send_to=MESSAGE_ROUTE_TO_NONE, ) @@ -190,3 +221,15 @@ class QaEngineer(Role): if not self.src_workspace: self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name return rsp + + async def _think(self) -> bool: + if not self.rc.news: + return False + msg = self.rc.news[0] + if msg.cause_by == any_to_str(SummarizeCode): + self.input_args = msg.instruct_content + self.repo = ProjectRepo(self.input_args.project_path) + if self.repo.src_relative_path is None: + path = get_project_srcs_path(self.repo.workdir) + self.repo.with_src_path(path) + return True diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1eaa77fa3..5592841eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -45,7 +45,6 @@ from metagpt.schema import ( ) from metagpt.strategy.planner import Planner from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator -from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output if TYPE_CHECKING: @@ -196,29 +195,6 @@ class Role(SerializationMixin, ContextMixin, BaseModel): value.context = self.context self.rc.todo = value - @property - def git_repo(self): - """Git repo""" - return self.context.git_repo - - @git_repo.setter - def git_repo(self, value): - self.context.git_repo = value - - @property - def src_workspace(self): - """Source workspace under git repo""" - return self.context.src_workspace - - @src_workspace.setter - def src_workspace(self, value): - self.context.src_workspace = value - - @property - def project_repo(self) -> ProjectRepo: - project_repo = ProjectRepo(self.context.git_repo) - return project_repo.with_src_path(self.context.src_workspace) if self.context.src_workspace else project_repo - @property def prompt_schema(self): """Prompt schema: json/markdown""" diff --git a/metagpt/schema.py b/metagpt/schema.py index 5af16bc38..5106fad4d 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -31,6 +31,7 @@ from pydantic import ( ConfigDict, Field, PrivateAttr, + create_model, field_serializer, field_validator, model_serializer, @@ -43,13 +44,18 @@ from metagpt.const import ( MESSAGE_ROUTE_FROM, MESSAGE_ROUTE_TO, MESSAGE_ROUTE_TO_ALL, - PRDS_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, ) from metagpt.logs import logger from metagpt.repo_parser import DotClassInfo -from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, import_class +from metagpt.utils.common import ( + CodeParser, + any_to_str, + any_to_str_set, + aread, + import_class, +) from metagpt.utils.exceptions import handle_exception from metagpt.utils.report import TaskReporter from metagpt.utils.serialize import ( @@ -157,6 +163,30 @@ class Document(BaseModel): def __repr__(self): return self.content + @classmethod + async def load( + cls, filename: Union[str, Path], project_path: Optional[Union[str, Path]] = None + ) -> Optional["Document"]: + """ + Load a document from a file. + + Args: + filename (Union[str, Path]): The path to the file to load. + project_path (Optional[Union[str, Path]], optional): The path to the project. Defaults to None. + + Returns: + Optional[Document]: The loaded document, or None if the file does not exist. + + """ + if not filename or not Path(filename).exists(): + return None + content = await aread(filename=filename) + doc = cls(content=content, filename=str(filename)) + if project_path and Path(filename).is_relative_to(project_path): + doc.root_path = Path(filename).relative_to(project_path).parent + doc.filename = Path(filename).name + return doc + class Documents(BaseModel): """A class representing a collection of documents. @@ -360,6 +390,22 @@ class Message(BaseModel): def add_metadata(self, key: str, value: str): self.metadata[key] = value + @staticmethod + def create_instruct_value(kvs: Dict[str, Any], class_name: str = "") -> BaseModel: + """ + Dynamically creates a Pydantic BaseModel subclass based on a given dictionary. + + Parameters: + - data: A dictionary from which to create the BaseModel subclass. + + Returns: + - A Pydantic BaseModel subclass instance populated with the given data. + """ + if not class_name: + class_name = "DM" + uuid.uuid4().hex[0:8] + dynamic_class = create_model(class_name, **{key: (value.__class__, ...) for key, value in kvs.items()}) + return dynamic_class.model_validate(kvs) + class UserMessage(Message): """便于支持OpenAI的消息 @@ -762,22 +808,6 @@ class CodePlanAndChangeContext(BaseModel): design_filename: str = "" task_filename: str = "" - @staticmethod - def loads(filenames: List, **kwargs) -> CodePlanAndChangeContext: - ctx = CodePlanAndChangeContext(requirement=kwargs.get("requirement", ""), issue=kwargs.get("issue", "")) - for filename in filenames: - filename = Path(filename) - if filename.is_relative_to(PRDS_FILE_REPO): - ctx.prd_filename = filename.name - continue - if filename.is_relative_to(SYSTEM_DESIGN_FILE_REPO): - ctx.design_filename = filename.name - continue - if filename.is_relative_to(TASK_FILE_REPO): - ctx.task_filename = filename.name - continue - return ctx - # mermaid class view class UMLClassMeta(BaseModel): diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index e2520ef13..6d40828af 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -899,3 +899,44 @@ async def init_python_folder(workdir: str | Path): return async with aiofiles.open(init_filename, "a"): os.utime(init_filename, None) + + +def get_markdown_code_block_type(filename: str) -> str: + if not filename: + return "" + ext = Path(filename).suffix + types = { + ".py": "python", + ".js": "javascript", + ".java": "java", + ".cpp": "cpp", + ".c": "c", + ".html": "html", + ".css": "css", + ".xml": "xml", + ".json": "json", + ".yaml": "yaml", + ".md": "markdown", + ".sql": "sql", + ".rb": "ruby", + ".php": "php", + ".sh": "bash", + ".swift": "swift", + ".go": "go", + ".rs": "rust", + ".pl": "perl", + ".asm": "assembly", + ".r": "r", + ".scss": "scss", + ".sass": "sass", + ".lua": "lua", + ".ts": "typescript", + ".tsx": "tsx", + ".jsx": "jsx", + ".yml": "yaml", + ".ini": "ini", + ".toml": "toml", + ".svg": "xml", # SVG can often be treated as XML + # Add more file extensions and corresponding code block types as needed + } + return types.get(ext, "") diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7b09c1775..f3d6350bd 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -156,6 +156,8 @@ class GitRepository: :param local_path: The local path to check. :return: True if the directory is a Git repository, False otherwise. """ + if not local_path: + return False git_dir = Path(local_path) / ".git" if git_dir.exists() and is_git_dir(git_dir): return True diff --git a/metagpt/utils/project_repo.py b/metagpt/utils/project_repo.py index 64ed602a9..5761c0188 100644 --- a/metagpt/utils/project_repo.py +++ b/metagpt/utils/project_repo.py @@ -140,10 +140,11 @@ class ProjectRepo(FileRepository): return bool(code_files) def with_src_path(self, path: str | Path) -> ProjectRepo: - try: - self._srcs_path = Path(path).relative_to(self.workdir) - except ValueError: - self._srcs_path = Path(path) + path = Path(path) + if path.is_relative_to(self.workdir): + self._srcs_path = path.relative_to(self.workdir) + else: + self._srcs_path = path return self @property diff --git a/tests/metagpt/actions/test_action_node.py b/tests/metagpt/actions/test_action_node.py index 989e2249c..bc85925a8 100644 --- a/tests/metagpt/actions/test_action_node.py +++ b/tests/metagpt/actions/test_action_node.py @@ -303,5 +303,4 @@ def test_action_node_from_pydantic_and_print_everything(): if __name__ == "__main__": - test_create_model_class() - test_create_model_class_with_mapping() + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/test_schema.py b/tests/metagpt/test_schema.py index 6f54b062d..48f13f4a2 100644 --- a/tests/metagpt/test_schema.py +++ b/tests/metagpt/test_schema.py @@ -392,5 +392,11 @@ async def test_parse_resources(context, content: str, key_descriptions): assert k in result +@pytest.mark.parametrize(("name", "value"), [("c1", {"age": 10, "name": "Alice"}), ("", {"path": __file__})]) +def test_create_instruct_value(name, value): + obj = Message.create_instruct_value(kvs=value, class_name=name) + assert obj.model_dump() == value + + if __name__ == "__main__": pytest.main([__file__, "-s"])