Merge branch 'feature/explit_io' into 'mgx_ops'

feat: Implemenet of RFC236 #3

See merge request pub/MetaGPT!110

metagpt/actions/action.py

@@ -22,7 +22,6 @@ from metagpt.schema import (
     SerializationMixin,
     TestingContext,
 )
-from metagpt.utils.project_repo import ProjectRepo
 
 
 class Action(SerializationMixin, ContextMixin, BaseModel):
@@ -36,12 +35,6 @@ class Action(SerializationMixin, ContextMixin, BaseModel):
     desc: str = ""  # for skill manager
     node: ActionNode = Field(default=None, exclude=True)
 
-    @property
-    def repo(self) -> ProjectRepo:
-        if not self.context.repo:
-            self.context.repo = ProjectRepo(self.context.git_repo)
-        return self.context.repo
-
     @property
     def prompt_schema(self):
         return self.config.prompt_schema

metagpt/actions/debug_error.py

@@ -9,13 +9,15 @@
         2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
 """
 import re
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from metagpt.actions.action import Action
 from metagpt.logs import logger
 from metagpt.schema import RunCodeContext, RunCodeResult
 from metagpt.utils.common import CodeParser
+from metagpt.utils.project_repo import ProjectRepo
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -47,6 +49,8 @@ Now you should start rewriting the code:
 
 class DebugError(Action):
     i_context: RunCodeContext = Field(default_factory=RunCodeContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     async def run(self, *args, **kwargs) -> str:
         output_doc = await self.repo.test_outputs.get(filename=self.i_context.output_filename)
@@ -59,9 +63,7 @@ class DebugError(Action):
             return ""
 
         logger.info(f"Debug and rewrite {self.i_context.test_filename}")
-        code_doc = await self.repo.with_src_path(self.context.src_workspace).srcs.get(
-            filename=self.i_context.code_filename
-        )
+        code_doc = await self.repo.srcs.get(filename=self.i_context.code_filename)
         if not code_doc:
             return ""
         test_doc = await self.repo.tests.get(filename=self.i_context.test_filename)

metagpt/actions/design_api.py

@@ -8,10 +8,14 @@
             1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
             2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
 @Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
+@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
 """
 import json
+import uuid
 from pathlib import Path
-from typing import Optional
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
 
 from metagpt.actions import Action
 from metagpt.actions.design_api_an import (
@@ -22,10 +26,17 @@ from metagpt.actions.design_api_an import (
     REFINED_DESIGN_NODE,
     REFINED_PROGRAM_CALL_FLOW,
 )
-from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
+from metagpt.const import (
+    DATA_API_DESIGN_FILE_REPO,
+    DEFAULT_WORKSPACE_ROOT,
+    SEQ_FLOW_FILE_REPO,
+)
 from metagpt.logs import logger
 from metagpt.schema import AIMessage, Document, Documents, Message
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import aread, awrite, to_markdown_code_block
 from metagpt.utils.mermaid import mermaid_to_file
+from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.report import DocsReporter, GalleryReporter
 
 NEW_REQ_TEMPLATE = """
@@ -37,6 +48,7 @@ NEW_REQ_TEMPLATE = """
 """
 
 
+@register_tool(tags=["software development", "write system design"])
 class WriteDesign(Action):
     name: str = ""
     i_context: Optional[str] = None
@@ -45,21 +57,134 @@ class WriteDesign(Action):
         "data structures, library tables, processes, and paths. Please provide your design, feedback "
         "clearly and in detail."
     )
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
-    async def run(self, with_messages: Message, schema: str = None):
-        # Use `git status` to identify which PRD documents have been modified in the `docs/prd` directory.
-        changed_prds = self.repo.docs.prd.changed_files
-        # Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone
-        # changes.
-        changed_system_designs = self.repo.docs.system_design.changed_files
+    async def run(
+        self,
+        with_messages: List[Message] = None,
+        *,
+        user_requirement: str = "",
+        prd_filename: str = "",
+        legacy_design_filename: str = "",
+        extra_info: str = "",
+        output_pathname: str = "",
+        **kwargs,
+    ) -> AIMessage:
+        """
+        Write a system design.
+
+        Args:
+            user_requirement (str): The user's requirements for the system design.
+            prd_filename (str, optional): The filename of the Product Requirement Document (PRD).
+            legacy_design_filename (str, optional): The filename of the legacy design document.
+            extra_info (str, optional): Additional information to be included in the system design.
+            output_pathname (str, optional): The output path name of file that the system design should be saved to.
+
+        Returns:
+            AIMessage: An AIMessage object containing the system design.
+
+        Example:
+            # Write a new system design.
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+
+            # Modify an exists system design.
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> legacy_design_filename = "/path/to/exists/design/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+
+            # Write a new system design with the given PRD(Product Requirement Document).
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> prd_filename = "/path/to/prd/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+
+            # Modify an exists system design with the given PRD(Product Requirement Document).
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> prd_filename = "/path/to/prd/filename"
+            >>> legacy_design_filename = "/path/to/exists/design/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, prd_filename=prd_filename)
+            >>> print(result.content)
+            TSystem Design filename: "/path/to/design/filename"
+
+            # Write a new system design and save to the path name.
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> output_pathname = "/path/to/design/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+
+            # Modify an exists system design and save to the path name.
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> legacy_design_filename = "/path/to/exists/design/filename"
+            >>> output_pathname = "/path/to/design/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, output_pathname=output_pathname)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+
+            # Write a new system design with the given PRD(Product Requirement Document) and save to the path name.
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> prd_filename = "/path/to/prd/filename"
+            >>> output_pathname = "/path/to/design/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, prd_filename=prd_filename, output_pathname=output_pathname)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+
+            # Modify an exists system design with the given PRD(Product Requirement Document) and save to the path name.
+            >>> user_requirement = "Your user requirements"
+            >>> extra_info = "Your extra information"
+            >>> prd_filename = "/path/to/prd/filename"
+            >>> legacy_design_filename = "/path/to/exists/design/filename"
+            >>> output_pathname = "/path/to/design/filename"
+            >>> action = WriteDesign()
+            >>> result = await action.run(user_requirement=user_requirement, extra_info=extra_info, legacy_design_filename=legacy_design_filename, prd_filename=prd_filename, output_pathname=output_pathname)
+            >>> print(result.content)
+            System Design filename: "/path/to/design/filename"
+        """
+        if not with_messages:
+            return await self._execute_api(
+                user_requirement=user_requirement,
+                prd_filename=prd_filename,
+                legacy_design_filename=legacy_design_filename,
+                extra_info=extra_info,
+                output_pathname=output_pathname,
+            )
+
+        self.input_args = with_messages[-1].instruct_content
+        self.repo = ProjectRepo(self.input_args.project_path)
+        changed_prds = self.input_args.changed_prd_filenames
+        changed_system_designs = [
+            str(self.repo.docs.system_design.workdir / i)
+            for i in list(self.repo.docs.system_design.changed_files.keys())
+        ]
 
         # For those PRDs and design documents that have undergone changes, regenerate the design content.
         changed_files = Documents()
-        for filename in changed_prds.keys():
+        for filename in changed_prds:
             doc = await self._update_system_design(filename=filename)
             changed_files.docs[filename] = doc
 
-        for filename in changed_system_designs.keys():
+        for filename in changed_system_designs:
             if filename in changed_files.docs:
                 continue
             doc = await self._update_system_design(filename=filename)
@@ -68,6 +193,11 @@ class WriteDesign(Action):
             logger.info("Nothing has changed.")
         # Wait until all files under `docs/system_designs/` are processed before sending the publish message,
         # leaving room for global optimization in subsequent steps.
+        kvs = self.input_args.model_dump()
+        kvs["changed_system_design_filenames"] = [
+            str(self.repo.docs.system_design.workdir / i)
+            for i in list(self.repo.docs.system_design.changed_files.keys())
+        ]
         return AIMessage(
             content="Designing is complete. "
             + "\n".join(
@@ -75,6 +205,7 @@ class WriteDesign(Action):
                 + list(self.repo.resources.data_api_design.changed_files.keys())
                 + list(self.repo.resources.seq_flow.changed_files.keys())
             ),
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteDesignOutput"),
             cause_by=self,
         )
 
@@ -89,14 +220,15 @@ class WriteDesign(Action):
         return system_design_doc
 
     async def _update_system_design(self, filename) -> Document:
-        prd = await self.repo.docs.prd.get(filename)
-        old_system_design_doc = await self.repo.docs.system_design.get(filename)
+        root_relative_path = Path(filename).relative_to(self.repo.workdir)
+        prd = await Document.load(filename=filename, project_path=self.repo.workdir)
+        old_system_design_doc = await self.repo.docs.system_design.get(root_relative_path.name)
         async with DocsReporter(enable_llm_stream=True) as reporter:
             await reporter.async_report({"type": "design"}, "meta")
             if not old_system_design_doc:
                 system_design = await self._new_system_design(context=prd.content)
                 doc = await self.repo.docs.system_design.save(
-                    filename=filename,
+                    filename=prd.filename,
                     content=system_design.instruct_content.model_dump_json(),
                     dependencies={prd.root_relative_path},
                 )
@@ -133,3 +265,40 @@ class WriteDesign(Action):
         image_path = pathname.parent / f"{pathname.name}.png"
         if image_path.exists():
             await GalleryReporter().async_report(image_path, "path")
+
+    async def _execute_api(
+        self,
+        user_requirement: str = "",
+        prd_filename: str = "",
+        legacy_design_filename: str = "",
+        extra_info: str = "",
+        output_pathname: str = "",
+    ) -> AIMessage:
+        prd_content = ""
+        if prd_filename:
+            prd_content = await aread(filename=prd_filename)
+        context = "### User Requirements\n{user_requirement}\n### Extra_info\n{extra_info}\n### PRD\n{prd}\n".format(
+            user_requirement=to_markdown_code_block(user_requirement),
+            extra_info=to_markdown_code_block(extra_info),
+            prd=to_markdown_code_block(prd_content),
+        )
+        if not legacy_design_filename:
+            node = await self._new_system_design(context=context)
+            design = Document(content=node.instruct_content.model_dump_json())
+        else:
+            old_design_content = await aread(filename=legacy_design_filename)
+            design = await self._merge(
+                prd_doc=Document(content=context), system_design_doc=Document(content=old_design_content)
+            )
+
+        if not output_pathname:
+            output_path = DEFAULT_WORKSPACE_ROOT
+            output_path.mkdir(parents=True, exist_ok=True)
+            output_pathname = Path(output_path) / f"{uuid.uuid4().hex}.json"
+        await awrite(filename=output_pathname, data=design.content)
+        kvs = {"changed_system_design_filenames": [output_pathname]}
+
+        return AIMessage(
+            content=f'System Design filename: "{str(output_pathname)}"',
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs),
+        )

metagpt/actions/prepare_documents.py

@@ -17,6 +17,7 @@ from metagpt.logs import logger
 from metagpt.schema import AIMessage
 from metagpt.utils.common import any_to_str
 from metagpt.utils.file_repository import FileRepository
+from metagpt.utils.project_repo import ProjectRepo
 
 
 class PrepareDocuments(Action):
@@ -36,7 +37,7 @@ class PrepareDocuments(Action):
     def config(self):
         return self.context.config
 
-    def _init_repo(self):
+    def _init_repo(self) -> ProjectRepo:
         """Initialize the Git environment."""
         if not self.config.project_path:
             name = self.config.project_name or FileRepository.new_filename()
@@ -45,8 +46,9 @@ class PrepareDocuments(Action):
             path = Path(self.config.project_path)
         if path.exists() and not self.config.inc:
             shutil.rmtree(path)
-        self.config.project_path = path
-        self.context.set_repo_dir(path)
+        self.context.kwargs.project_path = path
+        self.context.kwargs.inc = self.config.inc
+        return ProjectRepo(path)
 
     async def run(self, with_messages, **kwargs):
         """Create and initialize the workspace folder, initialize the Git environment."""
@@ -67,10 +69,22 @@ class PrepareDocuments(Action):
                     max_auto_summarize_code=0,
                 )
 
-        self._init_repo()
+        repo = self._init_repo()
 
         # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`.
-        doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
+        await repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
         # Send a Message notification to the WritePRD action, instructing it to process requirements using
         # `docs/requirement.txt` and `docs/prd/`.
-        return AIMessage(content="", instruct_content=doc, cause_by=self, send_to=self.send_to)
+        return AIMessage(
+            content="",
+            instruct_content=AIMessage.create_instruct_value(
+                kvs={
+                    "project_path": str(repo.workdir),
+                    "requirements_filename": str(repo.docs.workdir / REQUIREMENT_FILENAME),
+                    "prd_filenames": [str(repo.docs.prd.workdir / i) for i in repo.docs.prd.all_files],
+                },
+                class_name="PrepareDocumentsOutput",
+            ),
+            cause_by=self,
+            send_to=self.send_to,
+        )

metagpt/actions/project_management.py

@@ -8,16 +8,23 @@
         1. Divide the context into three components: legacy code, unit test code, and console log.
         2. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
         3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality.
+@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
 """
 
 import json
-from typing import Optional
+from pathlib import Path
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
 
 from metagpt.actions.action import Action
 from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE
 from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME
 from metagpt.logs import logger
-from metagpt.schema import AIMessage, Document, Documents
+from metagpt.schema import AIMessage, Document, Documents, Message
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import aread, to_markdown_code_block
+from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.report import DocsReporter
 
 NEW_REQ_TEMPLATE = """
@@ -29,19 +36,56 @@ NEW_REQ_TEMPLATE = """
 """
 
 
+@register_tool(tags=["software development", "write a project schedule given a project system design file"])
 class WriteTasks(Action):
     name: str = "CreateTasks"
     i_context: Optional[str] = None
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
-    async def run(self, with_messages):
-        changed_system_designs = self.repo.docs.system_design.changed_files
-        changed_tasks = self.repo.docs.task.changed_files
+    async def run(
+        self, with_messages: List[Message] = None, *, user_requirement: str = "", design_filename: str = "", **kwargs
+    ) -> AIMessage:
+        """
+        Write a project schedule given a project system design file.
+
+        Args:
+            user_requirement (str, optional): A string specifying the user's requirements. Defaults to an empty string.
+            design_filename (str): The filename of the project system design file. Defaults to an empty string.
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            AIMessage: The generated project schedule.
+
+        Example:
+            # Write a new project schedule.
+            >>> design_filename = "/path/to/design/filename"
+            >>> action = WriteTasks()
+            >>> result = await action.run(design_filename=design_filename)
+            >>> print(result.content)
+            The project schedule is balabala...
+
+            # Write a new project schedule with the user requirement.
+            >>> design_filename = "/path/to/design/filename"
+            >>> user_requirement = "Your user requirements"
+            >>> action = WriteTasks()
+            >>> result = await action.run(design_filename=design_filename, user_requirement=user_requirement)
+            >>> print(result.content)
+            The project schedule is balabala...
+        """
+        if not with_messages:
+            return await self._execute_api(user_requirement=user_requirement, design_filename=design_filename)
+
+        self.input_args = with_messages[-1].instruct_content
+        self.repo = ProjectRepo(self.input_args.project_path)
+        changed_system_designs = self.input_args.changed_system_design_filenames
+        changed_tasks = [str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())]
         change_files = Documents()
         # Rewrite the system designs that have undergone changes based on the git head diff under
         # `docs/system_designs/`.
         for filename in changed_system_designs:
             task_doc = await self._update_tasks(filename=filename)
-            change_files.docs[filename] = task_doc
+            change_files.docs[str(self.repo.docs.task.workdir / task_doc.filename)] = task_doc
 
         # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`.
         for filename in changed_tasks:
@@ -54,6 +98,11 @@ class WriteTasks(Action):
             logger.info("Nothing has changed.")
         # Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for
         # global optimization in subsequent steps.
+        kvs = self.input_args.model_dump()
+        kvs["changed_task_filenames"] = [
+            str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())
+        ]
+        kvs["python_package_dependency_filename"] = str(self.repo.workdir / PACKAGE_REQUIREMENTS_FILENAME)
         return AIMessage(
             content="WBS is completed. "
             + "\n".join(
@@ -61,12 +110,14 @@ class WriteTasks(Action):
                 + list(self.repo.docs.task.changed_files.keys())
                 + list(self.repo.resources.api_spec_and_task.changed_files.keys())
             ),
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTaskOutput"),
             cause_by=self,
         )
 
     async def _update_tasks(self, filename):
-        system_design_doc = await self.repo.docs.system_design.get(filename)
-        task_doc = await self.repo.docs.task.get(filename)
+        root_relative_path = Path(filename).relative_to(self.repo.workdir)
+        system_design_doc = await Document.load(filename=filename, project_path=self.repo.workdir)
+        task_doc = await self.repo.docs.task.get(root_relative_path.name)
         async with DocsReporter(enable_llm_stream=True) as reporter:
             await reporter.async_report({"type": "task"}, "meta")
             if task_doc:
@@ -75,7 +126,7 @@ class WriteTasks(Action):
             else:
                 rsp = await self._run_new_tasks(context=system_design_doc.content)
                 task_doc = await self.repo.docs.task.save(
-                    filename=filename,
+                    filename=system_design_doc.filename,
                     content=rsp.instruct_content.model_dump_json(),
                     dependencies={system_design_doc.root_relative_path},
                 )
@@ -84,7 +135,7 @@ class WriteTasks(Action):
             await reporter.async_report(self.repo.workdir / md.root_relative_path, "path")
         return task_doc
 
-    async def _run_new_tasks(self, context):
+    async def _run_new_tasks(self, context: str):
         node = await PM_NODE.fill(context, self.llm, schema=self.prompt_schema)
         return node
 
@@ -106,3 +157,11 @@ class WriteTasks(Action):
                 continue
             packages.add(pkg)
         await self.repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages))
+
+    async def _execute_api(self, user_requirement: str = "", design_filename: str = ""):
+        context = to_markdown_code_block(user_requirement)
+        if not design_filename:
+            content = await aread(filename=design_filename)
+            context += to_markdown_code_block(content)
+        node = await self._run_new_tasks(context)
+        return AIMessage(content=node.instruct_content.model_dump_json())

metagpt/actions/summarize_code.py

@@ -6,13 +6,16 @@
 @Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode.
 """
 from pathlib import Path
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action import Action
 from metagpt.logs import logger
 from metagpt.schema import CodeSummarizeContext
+from metagpt.utils.common import get_markdown_code_block_type
+from metagpt.utils.project_repo import ProjectRepo
 
 PROMPT_TEMPLATE = """
 NOTICE
@@ -90,6 +93,8 @@ flowchart TB
 class SummarizeCode(Action):
     name: str = "SummarizeCode"
     i_context: CodeSummarizeContext = Field(default_factory=CodeSummarizeContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     @retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60))
     async def summarize_code(self, prompt):
@@ -101,11 +106,10 @@ class SummarizeCode(Action):
         design_doc = await self.repo.docs.system_design.get(filename=design_pathname.name)
         task_pathname = Path(self.i_context.task_filename)
         task_doc = await self.repo.docs.task.get(filename=task_pathname.name)
-        src_file_repo = self.repo.with_src_path(self.context.src_workspace).srcs
         code_blocks = []
         for filename in self.i_context.codes_filenames:
-            code_doc = await src_file_repo.get(filename)
-            code_block = f"```python\n{code_doc.content}\n```\n-----"
+            code_doc = await self.repo.srcs.get(filename)
+            code_block = f"```{get_markdown_code_block_type(filename)}\n{code_doc.content}\n```\n---\n"
             code_blocks.append(code_block)
         format_example = FORMAT_EXAMPLE
         prompt = PROMPT_TEMPLATE.format(

metagpt/actions/write_code.py

@@ -16,17 +16,18 @@
 """
 
 import json
+from pathlib import Path
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions.action import Action
 from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
 from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE
-from metagpt.const import BUGFIX_FILENAME, REQUIREMENT_FILENAME
 from metagpt.logs import logger
 from metagpt.schema import CodingContext, Document, RunCodeResult
-from metagpt.utils.common import CodeParser
+from metagpt.utils.common import CodeParser, get_markdown_code_block_type
 from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.report import EditorReporter
 
@@ -44,9 +45,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 {task}
 
 ## Legacy Code
-```Code
 {code}
-```
 
 ## Debug logs
 ```text
@@ -61,14 +60,14 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 ```
 
 # Format example
-## Code: {filename}
+## Code: {demo_filename}.py
 ```python
-## {filename}
+## {demo_filename}.py
 ...
 ```
-## Code: {filename}
+## Code: {demo_filename}.js
 ```javascript
-// {filename}
+// {demo_filename}.js
 ...
 ```
 
@@ -89,6 +88,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
 class WriteCode(Action):
     name: str = "WriteCode"
     i_context: Document = Field(default_factory=Document)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
     async def write_code(self, prompt) -> str:
@@ -97,10 +98,16 @@ class WriteCode(Action):
         return code
 
     async def run(self, *args, **kwargs) -> CodingContext:
-        bug_feedback = await self.repo.docs.get(filename=BUGFIX_FILENAME)
+        bug_feedback = None
+        if self.input_args and hasattr(self.input_args, "issue_filename"):
+            bug_feedback = await Document.load(self.input_args.issue_filename)
         coding_context = CodingContext.loads(self.i_context.content)
+        if not coding_context.code_plan_and_change_doc:
+            coding_context.code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(
+                filename=coding_context.task_doc.filename
+            )
         test_doc = await self.repo.test_outputs.get(filename="test_" + coding_context.filename + ".json")
-        requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
+        requirement_doc = await Document.load(self.input_args.requirements_filename)
         summary_doc = None
         if coding_context.design_doc and coding_context.design_doc.filename:
             summary_doc = await self.repo.docs.code_summary.get(filename=coding_context.design_doc.filename)
@@ -109,29 +116,28 @@ class WriteCode(Action):
             test_detail = RunCodeResult.loads(test_doc.content)
             logs = test_detail.stderr
 
-        if bug_feedback:
-            code_context = coding_context.code_doc.content
-        elif self.config.inc:
+        if self.config.inc or bug_feedback:
             code_context = await self.get_codes(
                 coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo, use_inc=True
             )
         else:
             code_context = await self.get_codes(
-                coding_context.task_doc,
-                exclude=self.i_context.filename,
-                project_repo=self.repo.with_src_path(self.context.src_workspace),
+                coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo
             )
 
         if self.config.inc:
             prompt = REFINED_TEMPLATE.format(
                 user_requirement=requirement_doc.content if requirement_doc else "",
-                code_plan_and_change=str(coding_context.code_plan_and_change_doc),
+                code_plan_and_change=coding_context.code_plan_and_change_doc.content
+                if coding_context.code_plan_and_change_doc
+                else "",
                 design=coding_context.design_doc.content if coding_context.design_doc else "",
                 task=coding_context.task_doc.content if coding_context.task_doc else "",
                 code=code_context,
                 logs=logs,
                 feedback=bug_feedback.content if bug_feedback else "",
                 filename=self.i_context.filename,
+                demo_filename=Path(self.i_context.filename).stem,
                 summary_log=summary_doc.content if summary_doc else "",
             )
         else:
@@ -142,6 +148,7 @@ class WriteCode(Action):
                 logs=logs,
                 feedback=bug_feedback.content if bug_feedback else "",
                 filename=self.i_context.filename,
+                demo_filename=Path(self.i_context.filename).stem,
                 summary_log=summary_doc.content if summary_doc else "",
             )
         logger.info(f"Writing {coding_context.filename}..")
@@ -150,8 +157,9 @@ class WriteCode(Action):
             code = await self.write_code(prompt)
             if not coding_context.code_doc:
                 # avoid root_path pydantic ValidationError if use WriteCode alone
-                root_path = self.context.src_workspace if self.context.src_workspace else ""
-                coding_context.code_doc = Document(filename=coding_context.filename, root_path=str(root_path))
+                coding_context.code_doc = Document(
+                    filename=coding_context.filename, root_path=str(self.repo.src_relative_path)
+                )
             coding_context.code_doc.content = code
             await reporter.async_report(coding_context.code_doc, "document")
         return coding_context
@@ -178,35 +186,32 @@ class WriteCode(Action):
         code_filenames = m.get(TASK_LIST.key, []) if not use_inc else m.get(REFINED_TASK_LIST.key, [])
         codes = []
         src_file_repo = project_repo.srcs
-
         # Incremental development scenario
         if use_inc:
-            src_files = src_file_repo.all_files
-            # Get the old workspace contained the old codes and old workspace are created in previous CodePlanAndChange
-            old_file_repo = project_repo.git_repo.new_file_repository(relative_path=project_repo.old_workspace)
-            old_files = old_file_repo.all_files
-            # Get the union of the files in the src and old workspaces
-            union_files_list = list(set(src_files) | set(old_files))
-            for filename in union_files_list:
+            for filename in src_file_repo.all_files:
+                code_block_type = get_markdown_code_block_type(filename)
                 # Exclude the current file from the all code snippets
                 if filename == exclude:
                     # If the file is in the old workspace, use the old code
                     # Exclude unnecessary code to maintain a clean and focused main.py file, ensuring only relevant and
                     # essential functionality is included for the project’s requirements
-                    if filename in old_files and filename != "main.py":
+                    if filename != "main.py":
                         # Use old code
-                        doc = await old_file_repo.get(filename=filename)
+                        doc = await src_file_repo.get(filename=filename)
                     # If the file is in the src workspace, skip it
                     else:
                         continue
-                    codes.insert(0, f"-----Now, {filename} to be rewritten\n```{doc.content}```\n=====")
+                    codes.insert(
+                        0, f"### The name of file to rewrite: `{filename}`\n```{code_block_type}\n{doc.content}```\n"
+                    )
+                    logger.info(f"Prepare to rewrite `{filename}`")
                 # The code snippets are generated from the src workspace
                 else:
                     doc = await src_file_repo.get(filename=filename)
                     # If the file does not exist in the src workspace, skip it
                     if not doc:
                         continue
-                    codes.append(f"----- {filename}\n```{doc.content}```")
+                    codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n")
 
         # Normal scenario
         else:
@@ -217,6 +222,7 @@ class WriteCode(Action):
                 doc = await src_file_repo.get(filename=filename)
                 if not doc:
                     continue
-                codes.append(f"----- {filename}\n```{doc.content}```")
+                code_block_type = get_markdown_code_block_type(filename)
+                codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n")
 
         return "\n".join(codes)

metagpt/actions/write_code_plan_and_change_an.py

@@ -5,15 +5,16 @@
 @Author  : mannaandpoem
 @File    : write_code_plan_and_change_an.py
 """
-import os
-from typing import List
+from typing import List, Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 
 from metagpt.actions.action import Action
 from metagpt.actions.action_node import ActionNode
 from metagpt.logs import logger
-from metagpt.schema import CodePlanAndChangeContext
+from metagpt.schema import CodePlanAndChangeContext, Document
+from metagpt.utils.common import get_markdown_code_block_type
+from metagpt.utils.project_repo import ProjectRepo
 
 DEVELOPMENT_PLAN = ActionNode(
     key="Development Plan",
@@ -162,9 +163,8 @@ Role: You are a professional engineer; The main goal is to complete incremental
 {task}
 
 ## Legacy Code
-```Code
 {code}
-```
+
 
 ## Debug logs
 ```text
@@ -179,14 +179,14 @@ Role: You are a professional engineer; The main goal is to complete incremental
 ```
 
 # Format example
-## Code: {filename}
+## Code: {demo_filename}.py
 ```python
-## {filename}
+## {demo_filename}.py
 ...
 ```
-## Code: {filename}
+## Code: {demo_filename}.js
 ```javascript
-// {filename}
+// {demo_filename}.js
 ...
 ```
 
@@ -211,13 +211,15 @@ WRITE_CODE_PLAN_AND_CHANGE_NODE = ActionNode.from_children("WriteCodePlanAndChan
 class WriteCodePlanAndChange(Action):
     name: str = "WriteCodePlanAndChange"
     i_context: CodePlanAndChangeContext = Field(default_factory=CodePlanAndChangeContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     async def run(self, *args, **kwargs):
         self.llm.system_prompt = "You are a professional software engineer, your primary responsibility is to "
         "meticulously craft comprehensive incremental development plan and deliver detailed incremental change"
-        prd_doc = await self.repo.docs.prd.get(filename=self.i_context.prd_filename)
-        design_doc = await self.repo.docs.system_design.get(filename=self.i_context.design_filename)
-        task_doc = await self.repo.docs.task.get(filename=self.i_context.task_filename)
+        prd_doc = await Document.load(filename=self.i_context.prd_filename)
+        design_doc = await Document.load(filename=self.i_context.design_filename)
+        task_doc = await Document.load(filename=self.i_context.task_filename)
         context = CODE_PLAN_AND_CHANGE_CONTEXT.format(
             requirement=f"```text\n{self.i_context.requirement}\n```",
             issue=f"```text\n{self.i_context.issue}\n```",
@@ -230,8 +232,9 @@ class WriteCodePlanAndChange(Action):
         return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(context=context, llm=self.llm, schema="json")
 
     async def get_old_codes(self) -> str:
-        self.repo.old_workspace = self.repo.git_repo.workdir / os.path.basename(self.config.project_path)
-        old_file_repo = self.repo.git_repo.new_file_repository(relative_path=self.repo.old_workspace)
-        old_codes = await old_file_repo.get_all()
-        codes = [f"----- {code.filename}\n```{code.content}```" for code in old_codes]
+        old_codes = await self.repo.srcs.get_all()
+        codes = [
+            f"### File Name: `{code.filename}`\n```{get_markdown_code_block_type(code.filename)}\n{code.content}```\n"
+            for code in old_codes
+        ]
         return "\n".join(codes)

metagpt/actions/write_code_review.py

@@ -7,16 +7,17 @@
 @Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the
         WriteCode object, rather than passing them in when calling the run function.
 """
+from typing import Optional
 
-from pydantic import Field
+from pydantic import BaseModel, Field
 from tenacity import retry, stop_after_attempt, wait_random_exponential
 
 from metagpt.actions import WriteCode
 from metagpt.actions.action import Action
-from metagpt.const import REQUIREMENT_FILENAME
 from metagpt.logs import logger
-from metagpt.schema import CodingContext
+from metagpt.schema import CodingContext, Document
 from metagpt.utils.common import CodeParser
+from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.report import EditorReporter
 
 PROMPT_TEMPLATE = """
@@ -127,6 +128,8 @@ or
 class WriteCodeReview(Action):
     name: str = "WriteCodeReview"
     i_context: CodingContext = Field(default_factory=CodingContext)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
     async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, doc):
@@ -138,7 +141,9 @@ class WriteCodeReview(Action):
 
         # if LBTM, rewrite code
         async with EditorReporter(enable_llm_stream=True) as reporter:
-            await reporter.async_report({"type": "code", "filename": filename, "src_path": doc.root_relative_path}, "meta")
+            await reporter.async_report(
+                {"type": "code", "filename": filename, "src_path": doc.root_relative_path}, "meta"
+            )
             rewrite_prompt = f"{context_prompt}\n{cr_rsp}\n{REWRITE_CODE_TEMPLATE.format(filename=filename)}"
             code_rsp = await self._aask(rewrite_prompt)
             code = CodeParser.parse_code(text=code_rsp)
@@ -156,7 +161,7 @@ class WriteCodeReview(Action):
             code_context = await WriteCode.get_codes(
                 self.i_context.task_doc,
                 exclude=self.i_context.filename,
-                project_repo=self.repo.with_src_path(self.context.src_workspace),
+                project_repo=self.repo,
                 use_inc=self.config.inc,
             )
 
@@ -166,7 +171,7 @@ class WriteCodeReview(Action):
                 "## Code Files\n" + code_context + "\n",
             ]
             if self.config.inc:
-                requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
+                requirement_doc = await Document.load(filename=self.input_args.requirements_filename)
                 insert_ctx_list = [
                     "## User New Requirements\n" + str(requirement_doc) + "\n",
                     "## Code Plan And Change\n" + str(self.i_context.code_plan_and_change_doc) + "\n",

metagpt/actions/write_prd.py

@@ -9,12 +9,17 @@
             2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality.
             3. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
 @Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
+@Modified By: mashenquan, 2024/5/31. Implement Chapter 3 of RFC 236.
 """
 
 from __future__ import annotations
 
 import json
+import uuid
 from pathlib import Path
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
 
 from metagpt.actions import Action, ActionOutput
 from metagpt.actions.action_node import ActionNode
@@ -30,13 +35,16 @@ from metagpt.actions.write_prd_an import (
 from metagpt.const import (
     BUGFIX_FILENAME,
     COMPETITIVE_ANALYSIS_FILE_REPO,
+    DEFAULT_WORKSPACE_ROOT,
     REQUIREMENT_FILENAME,
 )
 from metagpt.logs import logger
 from metagpt.schema import AIMessage, Document, Documents, Message
-from metagpt.utils.common import CodeParser
+from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.common import CodeParser, aread, awrite, to_markdown_code_block
 from metagpt.utils.file_repository import FileRepository
 from metagpt.utils.mermaid import mermaid_to_file
+from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.report import DocsReporter, GalleryReporter
 
 CONTEXT_TEMPLATE = """
@@ -59,6 +67,7 @@ NEW_REQ_TEMPLATE = """
 """
 
 
+@register_tool(tags=["software development", "write product requirement documents"])
 class WritePRD(Action):
     """WritePRD deal with the following situations:
     1. Bugfix: If the requirement is a bugfix, the bugfix document will be generated.
@@ -66,10 +75,97 @@ class WritePRD(Action):
     3. Requirement update: If the requirement is an update, the PRD document will be updated.
     """
 
-    async def run(self, with_messages, *args, **kwargs) -> Message:
-        """Run the action."""
-        req: Document = await self.repo.requirement
-        docs: list[Document] = await self.repo.docs.prd.get_all()
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
+
+    async def run(
+        self,
+        with_messages: List[Message] = None,
+        *,
+        user_requirement: str = "",
+        output_pathname: str = "",
+        legacy_prd_filename: str = "",
+        extra_info: str = "",
+        **kwargs,
+    ) -> AIMessage:
+        """
+        Write a Product Requirement Document.
+
+        Args:
+            user_requirement (str): A string detailing the user's requirements.
+            output_pathname (str, optional): The path name of file that the output document should be saved to. Defaults to "".
+            legacy_prd_filename (str, optional): The file path of the legacy Product Requirement Document to use as a reference. Defaults to "".
+            extra_info (str, optional): Additional information to include in the document. Defaults to "".
+            **kwargs: Additional keyword arguments.
+
+        Returns:
+            AIMessage: The resulting message after generating the Product Requirement Document.
+
+        Example:
+            # Write a new PRD(Product Requirement Document)
+            >>> user_requirement = "YOUR REQUIREMENTS"
+            >>> extra_info = "YOUR EXTRA INFO"
+            >>> write_prd = WritePRD()
+            >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info)
+            >>> print(result.content)
+            PRD filename: "/path/to/prd/directory/213434ad.json"
+
+            # Modify a exists PRD(Product Requirement Document)
+            >>> user_requirement = "YOUR REQUIREMENTS"
+            >>> extra_info = "YOUR EXTRA INFO"
+            >>> legacy_prd_filename = "/path/to/exists/prd_filename"
+            >>> write_prd = WritePRD()
+            >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, legacy_prd_filename=legacy_prd_filename)
+            >>> print(result.content)
+            PRD filename: "/path/to/prd/directory/213434ad.json"
+
+            # Write and save a new PRD(Product Requirement Document) to the path name.
+            >>> user_requirement = "YOUR REQUIREMENTS"
+            >>> extra_info = "YOUR EXTRA INFO"
+            >>> output_pathname = "/path/to/prd/directory/213434ad.json"
+            >>> write_prd = WritePRD()
+            >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, output_pathname=output_pathname)
+            >>> print(result.content)
+            PRD filename: "/path/to/prd/directory/213434ad.json"
+
+            # Modify a exists PRD(Product Requirement Document) and save to the path name.
+            >>> user_requirement = "YOUR REQUIREMENTS"
+            >>> extra_info = "YOUR EXTRA INFO"
+            >>> legacy_prd_filename = "/path/to/exists/prd_filename"
+            >>> output_pathname = "/path/to/prd/directory/213434ad.json"
+            >>> write_prd = WritePRD()
+            >>> result = await write_prd.run(user_requirement=user_requirement, extra_info=extra_info, legacy_prd_filename=legacy_prd_filename, output_pathname=output_pathname)
+            >>> print(result.content)
+            PRD filename: "/path/to/prd/directory/213434ad.json"
+
+        """
+        if not with_messages:
+            return await self._execute_api(
+                user_requirement=user_requirement,
+                output_pathname=output_pathname,
+                legacy_prd_filename=legacy_prd_filename,
+                extra_info=extra_info,
+            )
+
+        self.input_args = with_messages[-1].instruct_content
+        if not self.input_args:
+            self.repo = ProjectRepo(self.context.kwargs.project_path)
+            await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[-1].content)
+            self.input_args = AIMessage.create_instruct_value(
+                kvs={
+                    "project_path": self.context.kwargs.project_path,
+                    "requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME),
+                    "prd_filenames": [str(self.repo.docs.prd.workdir / i) for i in self.repo.docs.prd.all_files],
+                },
+                class_name="PrepareDocumentsOutput",
+            )
+        else:
+            self.repo = ProjectRepo(self.input_args.project_path)
+        req = await Document.load(filename=self.input_args.requirements_filename)
+        docs: list[Document] = [
+            await Document.load(filename=i, project_path=self.repo.workdir) for i in self.input_args.prd_filenames
+        ]
+
         if not req:
             raise FileNotFoundError("No requirement document found.")
 
@@ -82,10 +178,18 @@ class WritePRD(Action):
         # if requirement is related to other documents, update them, otherwise create a new one
         if related_docs := await self.get_related_docs(req, docs):
             logger.info(f"Requirement update detected: {req.content}")
-            await self._handle_requirement_update(req, related_docs)
+            await self._handle_requirement_update(req=req, related_docs=related_docs)
         else:
             logger.info(f"New requirement detected: {req.content}")
             await self._handle_new_requirement(req)
+
+        kvs = self.input_args.model_dump()
+        kvs["changed_prd_filenames"] = [
+            str(self.repo.docs.prd.workdir / i) for i in list(self.repo.docs.prd.changed_files.keys())
+        ]
+        kvs["project_path"] = str(self.repo.workdir)
+        kvs["requirements_filename"] = str(self.repo.docs.workdir / REQUIREMENT_FILENAME)
+        self.context.kwargs.project_path = str(self.repo.workdir)
         return AIMessage(
             content="PRD is completed. "
             + "\n".join(
@@ -93,6 +197,7 @@ class WritePRD(Action):
                 + list(self.repo.resources.prd.changed_files.keys())
                 + list(self.repo.resources.competitive_analysis.changed_files.keys())
             ),
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WritePRDOutput"),
             cause_by=self,
         )
 
@@ -103,19 +208,31 @@ class WritePRD(Action):
         return AIMessage(
             content=f"A new issue is received: {BUGFIX_FILENAME}",
             cause_by=FixBug,
+            instruct_content=AIMessage.create_instruct_value(
+                {
+                    "project_path": str(self.repo.workdir),
+                    "issue_filename": str(self.repo.docs.workdir / BUGFIX_FILENAME),
+                    "requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME),
+                },
+                class_name="IssueDetail",
+            ),
             send_to="Alex",  # the name of Engineer
         )
 
+    async def _new_prd(self, requirement: str) -> ActionNode:
+        project_name = self.project_name
+        context = CONTEXT_TEMPLATE.format(requirements=requirement, project_name=project_name)
+        exclude = [PROJECT_NAME.key] if project_name else []
+        node = await WRITE_PRD_NODE.fill(
+            context=context, llm=self.llm, exclude=exclude, schema=self.prompt_schema
+        )  # schema=schema
+        return node
+
     async def _handle_new_requirement(self, req: Document) -> ActionOutput:
         """handle new requirement"""
         async with DocsReporter(enable_llm_stream=True) as reporter:
             await reporter.async_report({"type": "prd"}, "meta")
-            project_name = self.project_name
-            context = CONTEXT_TEMPLATE.format(requirements=req, project_name=project_name)
-            exclude = [PROJECT_NAME.key] if project_name else []
-            node = await WRITE_PRD_NODE.fill(
-                context=context, llm=self.llm, exclude=exclude, schema=self.prompt_schema
-            )  # schema=schema
+            node = await self._new_prd(req.content)
             await self._rename_workspace(node)
             new_prd_doc = await self.repo.docs.prd.save(
                 filename=FileRepository.new_filename() + ".json", content=node.instruct_content.model_dump_json()
@@ -128,7 +245,7 @@ class WritePRD(Action):
     async def _handle_requirement_update(self, req: Document, related_docs: list[Document]) -> ActionOutput:
         # ... requirement update logic ...
         for doc in related_docs:
-            await self._update_prd(req, doc)
+            await self._update_prd(req=req, prd_doc=doc)
         return Documents.from_iterable(documents=related_docs).to_action_output()
 
     async def _is_bugfix(self, context: str) -> bool:
@@ -159,7 +276,7 @@ class WritePRD(Action):
     async def _update_prd(self, req: Document, prd_doc: Document) -> Document:
         async with DocsReporter(enable_llm_stream=True) as reporter:
             await reporter.async_report({"type": "prd"}, "meta")
-            new_prd_doc: Document = await self._merge(req, prd_doc)
+            new_prd_doc: Document = await self._merge(req=req, related_doc=prd_doc)
             await self.repo.docs.prd.save_doc(doc=new_prd_doc)
             await self._save_competitive_analysis(new_prd_doc)
             md = await self.repo.resources.prd.save_pdf(doc=new_prd_doc)
@@ -186,4 +303,29 @@ class WritePRD(Action):
                 ws_name = CodeParser.parse_str(block="Project Name", text=prd)
             if ws_name:
                 self.project_name = ws_name
-        self.repo.git_repo.rename_root(self.project_name)
+        if self.repo:
+            self.repo.git_repo.rename_root(self.project_name)
+
+    async def _execute_api(
+        self, user_requirement: str, output_pathname: str, legacy_prd_filename: str, extra_info: str
+    ) -> AIMessage:
+        content = "#### User Requirements\n{user_requirement}\n#### Extra Info\n{extra_info}\n".format(
+            user_requirement=to_markdown_code_block(val=user_requirement),
+            extra_info=to_markdown_code_block(val=extra_info),
+        )
+        req = Document(content=content)
+        if not legacy_prd_filename:
+            node = await self._new_prd(requirement=req.content)
+            new_prd = Document(content=node.instruct_content.model_dump_json())
+        else:
+            content = await aread(filename=legacy_prd_filename)
+            old_prd = Document(content=content)
+            new_prd = await self._merge(req=req, related_doc=old_prd)
+
+        if not output_pathname:
+            output_path = DEFAULT_WORKSPACE_ROOT
+            output_path.mkdir(parents=True, exist_ok=True)
+            output_pathname = Path(output_path) / f"{uuid.uuid4().hex}.json"
+        await awrite(filename=output_pathname, data=new_prd.content)
+        kvs = AIMessage.create_instruct_value({"changed_prd_filenames": [str(output_pathname)]})
+        return AIMessage(content=f'PRD filename: "{str(output_pathname)}"', instruct_content=kvs)

metagpt/actions/write_prd_an.py

@@ -5,7 +5,7 @@
 @Author  : alexanderwu
 @File    : write_prd_an.py
 """
-from typing import List
+from typing import List, Union
 
 from metagpt.actions.action_node import ActionNode
 
@@ -132,7 +132,7 @@ REQUIREMENT_ANALYSIS = ActionNode(
 
 REFINED_REQUIREMENT_ANALYSIS = ActionNode(
     key="Refined Requirement Analysis",
-    expected_type=List[str],
+    expected_type=Union[List[str], str],
     instruction="Review and refine the existing requirement analysis into a string list to align with the evolving needs of the project "
     "due to incremental development. Ensure the analysis comprehensively covers the new features and enhancements "
     "required for the refined project scope.",

metagpt/context.py

@@ -8,7 +8,6 @@
 from __future__ import annotations
 
 import os
-from pathlib import Path
 from typing import Any, Dict, Optional
 
 from pydantic import BaseModel, ConfigDict
@@ -22,8 +21,6 @@ from metagpt.utils.cost_manager import (
     FireworksCostManager,
     TokenCostManager,
 )
-from metagpt.utils.git_repository import GitRepository
-from metagpt.utils.project_repo import ProjectRepo
 
 
 class AttrDict(BaseModel):
@@ -66,9 +63,6 @@ class Context(BaseModel):
     kwargs: AttrDict = AttrDict()
     config: Config = Config.default()
 
-    repo: Optional[ProjectRepo] = None
-    git_repo: Optional[GitRepository] = None
-    src_workspace: Optional[Path] = None
     cost_manager: CostManager = CostManager()
 
     _llm: Optional[BaseLLM] = None
@@ -80,11 +74,6 @@ class Context(BaseModel):
         # env.update({k: v for k, v in i.items() if isinstance(v, str)})
         return env
 
-    def set_repo_dir(self, path: str | Path):
-        repo_path = Path(path)
-        self.git_repo = GitRepository(local_path=repo_path, auto_init=True)
-        self.repo = ProjectRepo(self.git_repo)
-
     def _select_costmanager(self, llm_config: LLMConfig) -> CostManager:
         """Return a CostManager instance"""
         if llm_config.api_type == LLMType.FIREWORKS:
@@ -117,7 +106,6 @@ class Context(BaseModel):
             Dict[str, Any]: A dictionary containing serialized data.
         """
         return {
-            "workdir": str(self.repo.workdir) if self.repo else "",
             "kwargs": {k: v for k, v in self.kwargs.__dict__.items()},
             "cost_manager": self.cost_manager.model_dump_json(),
         }
@@ -130,13 +118,6 @@ class Context(BaseModel):
         """
         if not serialized_data:
             return
-        workdir = serialized_data.get("workdir")
-        if workdir:
-            self.git_repo = GitRepository(local_path=workdir, auto_init=True)
-            self.repo = ProjectRepo(self.git_repo)
-            src_workspace = self.git_repo.workdir / self.git_repo.workdir.name
-            if src_workspace.exists():
-                self.src_workspace = src_workspace
         kwargs = serialized_data.get("kwargs")
         if kwargs:
             for k, v in kwargs.items():

metagpt/environment/base_env.py

@@ -22,6 +22,7 @@ from metagpt.logs import logger
 from metagpt.memory import Memory
 from metagpt.schema import Message
 from metagpt.utils.common import get_function_schema, is_coroutine_func, is_send_to
+from metagpt.utils.git_repository import GitRepository
 
 if TYPE_CHECKING:
     from metagpt.roles.role import Role  # noqa: F401
@@ -243,8 +244,9 @@ class Environment(ExtEnv):
         self.member_addrs[obj] = addresses
 
     def archive(self, auto_archive=True):
-        if auto_archive and self.context.git_repo:
-            self.context.git_repo.archive()
+        if auto_archive and self.context.kwargs.get("project_path"):
+            git_repo = GitRepository(self.context.kwargs.project_path)
+            git_repo.archive()
 
     @classmethod
     def model_rebuild(cls, **kwargs):

metagpt/roles/architect.py

@@ -6,11 +6,9 @@
 @File    : architect.py
 """
 
-from metagpt.actions import UserRequirement, WritePRD
+from metagpt.actions import WritePRD
 from metagpt.actions.design_api import WriteDesign
-from metagpt.actions.prepare_documents import PrepareDocuments
 from metagpt.roles.role import Role
-from metagpt.utils.common import any_to_str
 
 
 class Architect(Role):
@@ -36,22 +34,7 @@ class Architect(Role):
         super().__init__(**kwargs)
         self.enable_memory = False
         # Initialize actions specific to the Architect role
-        self.set_actions([PrepareDocuments(send_to=any_to_str(self), context=self.context), WriteDesign])
+        self.set_actions([WriteDesign])
 
         # Set events or actions the Architect should watch or be aware of
-        self._watch({UserRequirement, PrepareDocuments, WritePRD})
-
-    async def _think(self) -> bool:
-        """Decide what to do"""
-        mappings = {
-            any_to_str(UserRequirement): 0,
-            any_to_str(PrepareDocuments): 1,
-            any_to_str(WritePRD): 1,
-        }
-        for i in self.rc.news:
-            idx = mappings.get(i.cause_by, -1)
-            if idx < 0:
-                continue
-            self.rc.todo = self.actions[idx]
-            return bool(self.rc.todo)
-        return False
+        self._watch({WritePRD})

metagpt/roles/engineer.py

@@ -24,20 +24,15 @@ from collections import defaultdict
 from pathlib import Path
 from typing import List, Optional, Set
 
-from metagpt.actions import (
-    Action,
-    UserRequirement,
-    WriteCode,
-    WriteCodeReview,
-    WriteTasks,
-)
+from pydantic import BaseModel, Field
+
+from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks
 from metagpt.actions.fix_bug import FixBug
 from metagpt.actions.prepare_documents import PrepareDocuments
 from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
 from metagpt.actions.summarize_code import SummarizeCode
 from metagpt.actions.write_code_plan_and_change_an import WriteCodePlanAndChange
 from metagpt.const import (
-    BUGFIX_FILENAME,
     CODE_PLAN_AND_CHANGE_FILE_REPO,
     MESSAGE_ROUTE_TO_SELF,
     REQUIREMENT_FILENAME,
@@ -63,6 +58,7 @@ from metagpt.utils.common import (
     init_python_folder,
 )
 from metagpt.utils.git_repository import ChangeType
+from metagpt.utils.project_repo import ProjectRepo
 
 IS_PASS_PROMPT = """
 {context}
@@ -100,23 +96,14 @@ class Engineer(Role):
     summarize_todos: list = []
     next_todo_action: str = ""
     n_summarize: int = 0
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
 
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
         self.enable_memory = False
         self.set_actions([WriteCode])
-        self._watch(
-            [
-                UserRequirement,
-                PrepareDocuments,
-                WriteTasks,
-                SummarizeCode,
-                WriteCode,
-                WriteCodeReview,
-                FixBug,
-                WriteCodePlanAndChange,
-            ]
-        )
+        self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug, WriteCodePlanAndChange])
         self.code_todos = []
         self.summarize_todos = []
         self.next_todo_action = any_to_name(WriteCode)
@@ -139,14 +126,20 @@ class Engineer(Role):
             coding_context = await todo.run()
             # Code review
             if review:
-                action = WriteCodeReview(i_context=coding_context, context=self.context, llm=self.llm)
+                action = WriteCodeReview(
+                    i_context=coding_context,
+                    repo=self.repo,
+                    input_args=self.input_args,
+                    context=self.context,
+                    llm=self.llm,
+                )
                 self._init_action(action)
                 coding_context = await action.run()
 
             dependencies = {coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}
             if self.config.inc:
                 dependencies.add(coding_context.code_plan_and_change_doc.root_relative_path)
-            await self.project_repo.srcs.save(
+            await self.repo.srcs.save(
                 filename=coding_context.filename,
                 dependencies=list(dependencies),
                 content=coding_context.code_doc.content,
@@ -186,9 +179,9 @@ class Engineer(Role):
             summary_filename = Path(todo.i_context.design_filename).with_suffix(".md").name
             dependencies = {todo.i_context.design_filename, todo.i_context.task_filename}
             for filename in todo.i_context.codes_filenames:
-                rpath = self.project_repo.src_relative_path / filename
+                rpath = self.repo.src_relative_path / filename
                 dependencies.add(str(rpath))
-            await self.project_repo.resources.code_summary.save(
+            await self.repo.resources.code_summary.save(
                 filename=summary_filename, content=summary, dependencies=dependencies
             )
             is_pass, reason = await self._is_pass(summary)
@@ -196,23 +189,39 @@ class Engineer(Role):
                 todo.i_context.reason = reason
                 tasks.append(todo.i_context.model_dump())
 
-                await self.project_repo.docs.code_summary.save(
+                await self.repo.docs.code_summary.save(
                     filename=Path(todo.i_context.design_filename).name,
                     content=todo.i_context.model_dump_json(),
                     dependencies=dependencies,
                 )
             else:
-                await self.project_repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name)
+                await self.repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name)
         self.summarize_todos = []
         logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}")
         if not tasks or self.config.max_auto_summarize_code == 0:
             self.n_summarize = 0
+            kvs = self.input_args.model_dump()
+            kvs["changed_src_filenames"] = [
+                str(self.repo.srcs.workdir / i) for i in list(self.repo.srcs.changed_files.keys())
+            ]
+            if self.repo.docs.code_plan_and_change.changed_files:
+                kvs["changed_code_plan_and_change_filenames"] = [
+                    str(self.repo.docs.code_plan_and_change.workdir / i)
+                    for i in list(self.repo.docs.code_plan_and_change.changed_files.keys())
+                ]
+            if self.repo.docs.code_summary.changed_files:
+                kvs["changed_code_summary_filenames"] = [
+                    str(self.repo.docs.code_summary.workdir / i)
+                    for i in list(self.repo.docs.code_summary.changed_files.keys())
+                ]
             return AIMessage(
-                content=f"Coding is complete. The source code is at {self.project_repo.workdir.name}/{self.project_repo.srcs.root_path}, containing: "
+                content=f"Coding is complete. The source code is at {self.repo.workdir.name}/{self.repo.srcs.root_path}, containing: "
                 + "\n".join(
-                    list(self.project_repo.resources.code_summary.changed_files.keys())
-                    + list(self.project_repo.srcs.changed_files.keys())
+                    list(self.repo.resources.code_summary.changed_files.keys())
+                    + list(self.repo.srcs.changed_files.keys())
+                    + list(self.repo.resources.code_plan_and_change.changed_files.keys())
                 ),
+                instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="SummarizeCodeOutput"),
                 cause_by=SummarizeCode,
                 send_to="Edward",  # The name of QaEngineer
             )
@@ -227,15 +236,15 @@ class Engineer(Role):
         code_plan_and_change = node.instruct_content.model_dump_json()
         dependencies = {
             REQUIREMENT_FILENAME,
-            str(self.project_repo.docs.prd.root_path / self.rc.todo.i_context.prd_filename),
-            str(self.project_repo.docs.system_design.root_path / self.rc.todo.i_context.design_filename),
-            str(self.project_repo.docs.task.root_path / self.rc.todo.i_context.task_filename),
+            str(Path(self.rc.todo.i_context.prd_filename).relative_to(self.repo.workdir)),
+            str(Path(self.rc.todo.i_context.design_filename).relative_to(self.repo.workdir)),
+            str(Path(self.rc.todo.i_context.task_filename).relative_to(self.repo.workdir)),
         }
         code_plan_and_change_filepath = Path(self.rc.todo.i_context.design_filename)
-        await self.project_repo.docs.code_plan_and_change.save(
+        await self.repo.docs.code_plan_and_change.save(
             filename=code_plan_and_change_filepath.name, content=code_plan_and_change, dependencies=dependencies
         )
-        await self.project_repo.resources.code_plan_and_change.save(
+        await self.repo.resources.code_plan_and_change.save(
             filename=code_plan_and_change_filepath.with_suffix(".md").name,
             content=node.content,
             dependencies=dependencies,
@@ -250,55 +259,49 @@ class Engineer(Role):
             return True, rsp
         return False, rsp
 
-    async def _think(self) -> Action | None:
+    async def _think(self) -> bool:
         if not self.rc.news:
-            return None
+            return False
         msg = self.rc.news[0]
-        if msg.cause_by == any_to_str(UserRequirement):
-            self.rc.todo = PrepareDocuments(
-                key_descriptions={
-                    "project_path": 'the project path if exists in "Original Requirement"',
-                    "src_filename": 'the file name of the source code file explicitly requested for modification if exists in "Original Requirement"',
-                },
-                context=self.context,
-                send_to=any_to_str(self),
-            )
-            return self.rc.todo
-
-        if not self.src_workspace:
-            self.src_workspace = get_project_srcs_path(self.project_repo.workdir)
+        input_args = msg.instruct_content
+        if msg.cause_by in {any_to_str(WriteTasks), any_to_str(FixBug)}:
+            self.input_args = input_args
+            self.repo = ProjectRepo(input_args.project_path)
+            if self.repo.src_relative_path is None:
+                path = get_project_srcs_path(self.repo.workdir)
+                self.repo.with_src_path(path)
         write_plan_and_change_filters = any_to_str_set([PrepareDocuments, WriteTasks, FixBug])
         write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode])
         summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview])
         if self.config.inc and msg.cause_by in write_plan_and_change_filters:
             logger.debug(f"TODO WriteCodePlanAndChange:{msg.model_dump_json()}")
             await self._new_code_plan_and_change_action(cause_by=msg.cause_by)
-            return self.rc.todo
+            return bool(self.rc.todo)
         if msg.cause_by in write_code_filters:
             logger.debug(f"TODO WriteCode:{msg.model_dump_json()}")
             await self._new_code_actions()
-            return self.rc.todo
+            return bool(self.rc.todo)
         if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self):
             logger.debug(f"TODO SummarizeCode:{msg.model_dump_json()}")
             await self._new_summarize_actions()
-            return self.rc.todo
-        return None
+            return bool(self.rc.todo)
+        return False
 
     async def _new_coding_context(self, filename, dependency) -> Optional[CodingContext]:
-        old_code_doc = await self.project_repo.srcs.get(filename)
+        old_code_doc = await self.repo.srcs.get(filename)
         if not old_code_doc:
-            old_code_doc = Document(root_path=str(self.project_repo.src_relative_path), filename=filename, content="")
+            old_code_doc = Document(root_path=str(self.repo.src_relative_path), filename=filename, content="")
         dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)}
         task_doc = None
         design_doc = None
         code_plan_and_change_doc = await self._get_any_code_plan_and_change() if await self._is_fixbug() else None
         for i in dependencies:
             if str(i.parent) == TASK_FILE_REPO:
-                task_doc = await self.project_repo.docs.task.get(i.name)
+                task_doc = await self.repo.docs.task.get(i.name)
             elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO:
-                design_doc = await self.project_repo.docs.system_design.get(i.name)
+                design_doc = await self.repo.docs.system_design.get(i.name)
             elif str(i.parent) == CODE_PLAN_AND_CHANGE_FILE_REPO:
-                code_plan_and_change_doc = await self.project_repo.docs.code_plan_and_change.get(i.name)
+                code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(i.name)
         if not task_doc or not design_doc:
             if filename == "__init__.py":  # `__init__.py` created by `init_python_folder`
                 return None
@@ -318,34 +321,66 @@ class Engineer(Role):
         if not context:
             return None  # `__init__.py` created by `init_python_folder`
         coding_doc = Document(
-            root_path=str(self.project_repo.src_relative_path), filename=filename, content=context.model_dump_json()
+            root_path=str(self.repo.src_relative_path), filename=filename, content=context.model_dump_json()
         )
         return coding_doc
 
     async def _new_code_actions(self):
         bug_fix = await self._is_fixbug()
         # Prepare file repos
-        changed_src_files = self.project_repo.srcs.changed_files
+        changed_src_files = self.repo.srcs.changed_files
         if self.context.kwargs.src_filename:
             changed_src_files = {self.context.kwargs.src_filename: ChangeType.UNTRACTED}
         if bug_fix:
-            changed_src_files = self.project_repo.srcs.all_files
-        changed_task_files = self.project_repo.docs.task.changed_files
+            changed_src_files = self.repo.srcs.all_files
         changed_files = Documents()
         # Recode caused by upstream changes.
-        for filename in changed_task_files:
-            design_doc = await self.project_repo.docs.system_design.get(filename)
-            task_doc = await self.project_repo.docs.task.get(filename)
-            code_plan_and_change_doc = await self.project_repo.docs.code_plan_and_change.get(filename)
+        if hasattr(self.input_args, "changed_task_filenames"):
+            changed_task_filenames = self.input_args.changed_task_filenames
+        else:
+            changed_task_filenames = [
+                str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())
+            ]
+        for filename in changed_task_filenames:
+            task_filename = Path(filename)
+            design_filename = None
+            if hasattr(self.input_args, "changed_system_design_filenames"):
+                changed_system_design_filenames = self.input_args.changed_system_design_filenames
+            else:
+                changed_system_design_filenames = [
+                    str(self.repo.docs.system_design.workdir / i)
+                    for i in list(self.repo.docs.system_design.changed_files.keys())
+                ]
+            for i in changed_system_design_filenames:
+                if task_filename.name == Path(i).name:
+                    design_filename = Path(i)
+                    break
+            code_plan_and_change_filename = None
+            if hasattr(self.input_args, "changed_code_plan_and_change_filenames"):
+                changed_code_plan_and_change_filenames = self.input_args.changed_code_plan_and_change_filenames
+            else:
+                changed_code_plan_and_change_filenames = [
+                    str(self.repo.docs.code_plan_and_change.workdir / i)
+                    for i in list(self.repo.docs.code_plan_and_change.changed_files.keys())
+                ]
+            for i in changed_code_plan_and_change_filenames:
+                if task_filename.name == Path(i).name:
+                    code_plan_and_change_filename = Path(i)
+                    break
+            design_doc = await Document.load(filename=design_filename, project_path=self.repo.workdir)
+            task_doc = await Document.load(filename=task_filename, project_path=self.repo.workdir)
+            code_plan_and_change_doc = await Document.load(
+                filename=code_plan_and_change_filename, project_path=self.repo.workdir
+            )
             task_list = self._parse_tasks(task_doc)
             await self._init_python_folder(task_list)
             for task_filename in task_list:
                 if self.context.kwargs.src_filename and task_filename != self.context.kwargs.src_filename:
                     continue
-                old_code_doc = await self.project_repo.srcs.get(task_filename)
+                old_code_doc = await self.repo.srcs.get(task_filename)
                 if not old_code_doc:
                     old_code_doc = Document(
-                        root_path=str(self.project_repo.src_relative_path), filename=task_filename, content=""
+                        root_path=str(self.repo.src_relative_path), filename=task_filename, content=""
                     )
                 if not code_plan_and_change_doc:
                     context = CodingContext(
@@ -360,7 +395,7 @@ class Engineer(Role):
                         code_plan_and_change_doc=code_plan_and_change_doc,
                     )
                 coding_doc = Document(
-                    root_path=str(self.project_repo.src_relative_path),
+                    root_path=str(self.repo.src_relative_path),
                     filename=task_filename,
                     content=context.model_dump_json(),
                 )
@@ -371,10 +406,11 @@ class Engineer(Role):
                     )
                 changed_files.docs[task_filename] = coding_doc
         self.code_todos = [
-            WriteCode(i_context=i, context=self.context, llm=self.llm) for i in changed_files.docs.values()
+            WriteCode(i_context=i, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm)
+            for i in changed_files.docs.values()
         ]
         # Code directly modified by the user.
-        dependency = await self.git_repo.get_dependency()
+        dependency = await self.repo.git_repo.get_dependency()
         for filename in changed_src_files:
             if filename in changed_files.docs:
                 continue
@@ -382,24 +418,30 @@ class Engineer(Role):
             if not coding_doc:
                 continue  # `__init__.py` created by `init_python_folder`
             changed_files.docs[filename] = coding_doc
-            self.code_todos.append(WriteCode(i_context=coding_doc, context=self.context, llm=self.llm))
+            self.code_todos.append(
+                WriteCode(
+                    i_context=coding_doc, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm
+                )
+            )
 
         if self.code_todos:
             self.set_todo(self.code_todos[0])
 
     async def _new_summarize_actions(self):
-        src_files = self.project_repo.srcs.all_files
+        src_files = self.repo.srcs.all_files
         # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc).
         summarizations = defaultdict(list)
         for filename in src_files:
-            dependencies = await self.project_repo.srcs.get_dependency(filename=filename)
+            dependencies = await self.repo.srcs.get_dependency(filename=filename)
             ctx = CodeSummarizeContext.loads(filenames=list(dependencies))
             summarizations[ctx].append(filename)
         for ctx, filenames in summarizations.items():
             if not ctx.design_filename or not ctx.task_filename:
                 continue  # cause by `__init__.py` which is created by `init_python_folder`
             ctx.codes_filenames = filenames
-            new_summarize = SummarizeCode(i_context=ctx, context=self.context, llm=self.llm)
+            new_summarize = SummarizeCode(
+                i_context=ctx, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm
+            )
             for i, act in enumerate(self.summarize_todos):
                 if act.i_context.task_filename == new_summarize.i_context.task_filename:
                     self.summarize_todos[i] = new_summarize
@@ -412,16 +454,37 @@ class Engineer(Role):
 
     async def _new_code_plan_and_change_action(self, cause_by: str):
         """Create a WriteCodePlanAndChange action for subsequent to-do actions."""
-        files = self.project_repo.all_files
         options = {}
         if cause_by != any_to_str(FixBug):
-            requirement_doc = await self.project_repo.docs.get(REQUIREMENT_FILENAME)
+            requirement_doc = await Document.load(filename=self.input_args.requirements_filename)
             options["requirement"] = requirement_doc.content
         else:
-            fixbug_doc = await self.project_repo.docs.get(BUGFIX_FILENAME)
+            fixbug_doc = await Document.load(filename=self.input_args.issue_filename)
             options["issue"] = fixbug_doc.content
-        code_plan_and_change_ctx = CodePlanAndChangeContext.loads(files, **options)
-        self.rc.todo = WriteCodePlanAndChange(i_context=code_plan_and_change_ctx, context=self.context, llm=self.llm)
+        # The code here is flawed: if there are multiple unrelated requirements, this piece of logic will break
+        if hasattr(self.input_args, "changed_prd_filenames"):
+            code_plan_and_change_ctx = CodePlanAndChangeContext(
+                requirement=options.get("requirement", ""),
+                issue=options.get("issue", ""),
+                prd_filename=self.input_args.changed_prd_filenames[0],
+                design_filename=self.input_args.changed_system_design_filenames[0],
+                task_filename=self.input_args.changed_task_filenames[0],
+            )
+        else:
+            code_plan_and_change_ctx = CodePlanAndChangeContext(
+                requirement=options.get("requirement", ""),
+                issue=options.get("issue", ""),
+                prd_filename=str(self.repo.docs.prd.workdir / self.repo.docs.prd.all_files[0]),
+                design_filename=str(self.repo.docs.system_design.workdir / self.repo.docs.system_design.all_files[0]),
+                task_filename=str(self.repo.docs.task.workdir / self.repo.docs.task.all_files[0]),
+            )
+        self.rc.todo = WriteCodePlanAndChange(
+            i_context=code_plan_and_change_ctx,
+            repo=self.repo,
+            input_args=self.input_args,
+            context=self.context,
+            llm=self.llm,
+        )
 
     @property
     def action_description(self) -> str:
@@ -433,17 +496,16 @@ class Engineer(Role):
             filename = Path(i)
             if filename.suffix != ".py":
                 continue
-            workdir = self.src_workspace / filename.parent
+            workdir = self.repo.srcs.workdir / filename.parent
             await init_python_folder(workdir)
 
     async def _is_fixbug(self) -> bool:
-        fixbug_doc = await self.project_repo.docs.get(BUGFIX_FILENAME)
-        return bool(fixbug_doc and fixbug_doc.content)
+        return bool(self.input_args and hasattr(self.input_args, "issue_filename"))
 
     async def _get_any_code_plan_and_change(self) -> Optional[Document]:
-        changed_files = self.project_repo.docs.code_plan_and_change.changed_files
+        changed_files = self.repo.docs.code_plan_and_change.changed_files
         for filename in changed_files.keys():
-            doc = await self.project_repo.docs.code_plan_and_change.get(filename)
+            doc = await self.repo.docs.code_plan_and_change.get(filename)
             if doc and doc.content:
                 return doc
         return None

metagpt/roles/product_manager.py

@@ -7,10 +7,12 @@
 @Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135.
 """
 
+
 from metagpt.actions import UserRequirement, WritePRD
 from metagpt.actions.prepare_documents import PrepareDocuments
 from metagpt.roles.role import Role, RoleReactMode
 from metagpt.utils.common import any_to_name, any_to_str
+from metagpt.utils.git_repository import GitRepository
 
 
 class ProductManager(Role):
@@ -40,7 +42,7 @@ class ProductManager(Role):
 
     async def _think(self) -> bool:
         """Decide what to do"""
-        if self.git_repo and not self.config.git_reinit:
+        if GitRepository.is_git_dir(self.config.project_path) and not self.config.git_reinit:
             self._set_state(1)
         else:
             self._set_state(0)

metagpt/roles/project_manager.py

@@ -6,11 +6,9 @@
 @File    : project_manager.py
 """
 
-from metagpt.actions import UserRequirement, WriteTasks
+from metagpt.actions import WriteTasks
 from metagpt.actions.design_api import WriteDesign
-from metagpt.actions.prepare_documents import PrepareDocuments
 from metagpt.roles.role import Role
-from metagpt.utils.common import any_to_str
 
 
 class ProjectManager(Role):
@@ -35,20 +33,5 @@ class ProjectManager(Role):
     def __init__(self, **kwargs) -> None:
         super().__init__(**kwargs)
         self.enable_memory = False
-        self.set_actions([PrepareDocuments(send_to=any_to_str(self), context=self.context), WriteTasks])
-        self._watch([UserRequirement, PrepareDocuments, WriteDesign])
-
-    async def _think(self) -> bool:
-        """Decide what to do"""
-        mappings = {
-            any_to_str(UserRequirement): 0,
-            any_to_str(PrepareDocuments): 1,
-            any_to_str(WriteDesign): 1,
-        }
-        for i in self.rc.news:
-            idx = mappings.get(i.cause_by, -1)
-            if idx < 0:
-                continue
-            self.rc.todo = self.actions[idx]
-            return bool(self.rc.todo)
-        return False
+        self.set_actions([WriteTasks])
+        self._watch([WriteDesign])

metagpt/roles/qa_engineer.py

@@ -14,6 +14,9 @@
 @Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
     of SummarizeCode.
 """
+from typing import Optional
+
+from pydantic import BaseModel, Field
 
 from metagpt.actions import DebugError, RunCode, UserRequirement, WriteTest
 from metagpt.actions.prepare_documents import PrepareDocuments
@@ -25,9 +28,11 @@ from metagpt.schema import AIMessage, Document, Message, RunCodeContext, Testing
 from metagpt.utils.common import (
     any_to_str,
     any_to_str_set,
+    get_project_srcs_path,
     init_python_folder,
     parse_recipient,
 )
+from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.report import EditorReporter
 
 
@@ -41,6 +46,8 @@ class QaEngineer(Role):
     )
     test_round_allowed: int = 5
     test_round: int = 0
+    repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
+    input_args: Optional[BaseModel] = Field(default=None, exclude=True)
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
@@ -48,31 +55,26 @@ class QaEngineer(Role):
 
         # FIXME: a bit hack here, only init one action to circumvent _think() logic,
         #  will overwrite _think() in future updates
-        self.set_actions(
-            [
-                WriteTest,
-            ]
-        )
-        self._watch([UserRequirement, PrepareDocuments, SummarizeCode, WriteTest, RunCode, DebugError])
+        self.set_actions([WriteTest])
+        self._watch([SummarizeCode, WriteTest, RunCode, DebugError])
         self.test_round = 0
 
     async def _write_test(self, message: Message) -> None:
-        src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs
         reqa_file = self.context.kwargs.reqa_file or self.config.reqa_file
-        changed_files = {reqa_file} if reqa_file else set(src_file_repo.changed_files.keys())
+        changed_files = {reqa_file} if reqa_file else set(self.repo.srcs.changed_files.keys())
         for filename in changed_files:
             # write tests
             if not filename or "test" in filename:
                 continue
-            code_doc = await src_file_repo.get(filename)
-            if not code_doc:
+            code_doc = await self.repo.srcs.get(filename)
+            if not code_doc or not code_doc.content:
                 continue
             if not code_doc.filename.endswith(".py"):
                 continue
-            test_doc = await self.project_repo.tests.get("test_" + code_doc.filename)
+            test_doc = await self.repo.tests.get("test_" + code_doc.filename)
             if not test_doc:
                 test_doc = Document(
-                    root_path=str(self.project_repo.tests.root_path), filename="test_" + code_doc.filename, content=""
+                    root_path=str(self.repo.tests.root_path), filename="test_" + code_doc.filename, content=""
                 )
             logger.info(f"Writing {test_doc.filename}..")
             context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc)
@@ -81,40 +83,38 @@ class QaEngineer(Role):
             async with EditorReporter(enable_llm_stream=True) as reporter:
                 await reporter.async_report({"type": "test", "filename": test_doc.filename}, "meta")
 
-                doc = await self.project_repo.tests.save_doc(
+                doc = await self.repo.tests.save_doc(
                     doc=context.test_doc, dependencies={context.code_doc.root_relative_path}
                 )
-                await reporter.async_report(self.project_repo.workdir / doc.root_relative_path, "path")
+                await reporter.async_report(self.repo.workdir / doc.root_relative_path, "path")
 
             # prepare context for run tests in next round
             run_code_context = RunCodeContext(
                 command=["python", context.test_doc.root_relative_path],
                 code_filename=context.code_doc.filename,
                 test_filename=context.test_doc.filename,
-                working_directory=str(self.project_repo.workdir),
-                additional_python_paths=[str(self.context.src_workspace)],
+                working_directory=str(self.repo.workdir),
+                additional_python_paths=[str(self.repo.srcs.workdir)],
             )
             self.publish_message(
                 AIMessage(content=run_code_context.model_dump_json(), cause_by=WriteTest, send_to=MESSAGE_ROUTE_TO_SELF)
             )
 
-        logger.info(f"Done {str(self.project_repo.tests.workdir)} generating.")
+        logger.info(f"Done {str(self.repo.tests.workdir)} generating.")
 
     async def _run_code(self, msg):
         run_code_context = RunCodeContext.loads(msg.content)
-        src_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get(
-            run_code_context.code_filename
-        )
+        src_doc = await self.repo.srcs.get(run_code_context.code_filename)
         if not src_doc:
             return
-        test_doc = await self.project_repo.tests.get(run_code_context.test_filename)
+        test_doc = await self.repo.tests.get(run_code_context.test_filename)
         if not test_doc:
             return
         run_code_context.code = src_doc.content
         run_code_context.test_code = test_doc.content
         result = await RunCode(i_context=run_code_context, context=self.context, llm=self.llm).run()
         run_code_context.output_filename = run_code_context.test_filename + ".json"
-        await self.project_repo.test_outputs.save(
+        await self.repo.test_outputs.save(
             filename=run_code_context.output_filename,
             content=result.model_dump_json(),
             dependencies={src_doc.root_relative_path, test_doc.root_relative_path},
@@ -124,31 +124,53 @@ class QaEngineer(Role):
         # the recipient might be Engineer or myself
         recipient = parse_recipient(result.summary)
         mappings = {"Engineer": "Alex", "QaEngineer": "Edward"}
-        self.publish_message(
-            AIMessage(
-                content=run_code_context.model_dump_json(),
-                cause_by=RunCode,
-                send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE),
+        if recipient != "Engineer":
+            self.publish_message(
+                AIMessage(
+                    content=run_code_context.model_dump_json(),
+                    cause_by=RunCode,
+                    instruct_content=self.input_args,
+                    send_to=MESSAGE_ROUTE_TO_SELF,
+                )
+            )
+        else:
+            kvs = self.input_args.model_dump()
+            kvs["changed_test_filenames"] = [
+                str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys())
+            ]
+            self.publish_message(
+                AIMessage(
+                    content=run_code_context.model_dump_json(),
+                    cause_by=RunCode,
+                    instruct_content=self.input_args,
+                    send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE),
+                )
             )
-        )
 
     async def _debug_error(self, msg):
         run_code_context = RunCodeContext.loads(msg.content)
-        code = await DebugError(i_context=run_code_context, context=self.context, llm=self.llm).run()
-        await self.project_repo.tests.save(filename=run_code_context.test_filename, content=code)
+        code = await DebugError(
+            i_context=run_code_context, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm
+        ).run()
+        await self.repo.tests.save(filename=run_code_context.test_filename, content=code)
         run_code_context.output = None
         self.publish_message(
             AIMessage(content=run_code_context.model_dump_json(), cause_by=DebugError, send_to=MESSAGE_ROUTE_TO_SELF)
         )
 
     async def _act(self) -> Message:
-        if self.project_path:
-            await init_python_folder(self.project_repo.tests.workdir)
+        if self.input_args.project_path:
+            await init_python_folder(self.repo.tests.workdir)
         if self.test_round > self.test_round_allowed:
+            kvs = self.input_args.model_dump()
+            kvs["changed_test_filenames"] = [
+                str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys())
+            ]
             result_msg = AIMessage(
                 content=f"Exceeding {self.test_round_allowed} rounds of tests, stop. "
-                + "\n".join(list(self.project_repo.tests.changed_files.keys())),
+                + "\n".join(list(self.repo.tests.changed_files.keys())),
                 cause_by=WriteTest,
+                instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTestOutput"),
                 send_to=MESSAGE_ROUTE_TO_NONE,
             )
             return result_msg
@@ -171,8 +193,13 @@ class QaEngineer(Role):
             elif msg.cause_by == any_to_str(UserRequirement):
                 return await self._parse_user_requirement(msg)
         self.test_round += 1
+        kvs = self.input_args.model_dump()
+        kvs["changed_test_filenames"] = [
+            str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys())
+        ]
         return AIMessage(
             content=f"Round {self.test_round} of tests done",
+            instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTestOutput"),
             cause_by=WriteTest,
             send_to=MESSAGE_ROUTE_TO_NONE,
         )
@@ -190,3 +217,15 @@ class QaEngineer(Role):
         if not self.src_workspace:
             self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name
         return rsp
+
+    async def _think(self) -> bool:
+        if not self.rc.news:
+            return False
+        msg = self.rc.news[0]
+        if msg.cause_by == any_to_str(SummarizeCode):
+            self.input_args = msg.instruct_content
+            self.repo = ProjectRepo(self.input_args.project_path)
+            if self.repo.src_relative_path is None:
+                path = get_project_srcs_path(self.repo.workdir)
+                self.repo.with_src_path(path)
+        return True

metagpt/roles/role.py

@@ -45,7 +45,6 @@ from metagpt.schema import (
 )
 from metagpt.strategy.planner import Planner
 from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator
-from metagpt.utils.project_repo import ProjectRepo
 from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output
 
 if TYPE_CHECKING:
@@ -196,29 +195,6 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
             value.context = self.context
         self.rc.todo = value
 
-    @property
-    def git_repo(self):
-        """Git repo"""
-        return self.context.git_repo
-
-    @git_repo.setter
-    def git_repo(self, value):
-        self.context.git_repo = value
-
-    @property
-    def src_workspace(self):
-        """Source workspace under git repo"""
-        return self.context.src_workspace
-
-    @src_workspace.setter
-    def src_workspace(self, value):
-        self.context.src_workspace = value
-
-    @property
-    def project_repo(self) -> ProjectRepo:
-        project_repo = ProjectRepo(self.context.git_repo)
-        return project_repo.with_src_path(self.context.src_workspace) if self.context.src_workspace else project_repo
-
     @property
     def prompt_schema(self):
         """Prompt schema: json/markdown"""
@@ -410,8 +386,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
             msg = response
         else:
             msg = AIMessage(content=response or "", cause_by=self.rc.todo, sent_from=self)
-        if self.enable_memory:
-            self.rc.memory.add(msg)
+        self.rc.memory.add(msg)
 
         return msg
 

metagpt/schema.py

@@ -31,6 +31,7 @@ from pydantic import (
     ConfigDict,
     Field,
     PrivateAttr,
+    create_model,
     field_serializer,
     field_validator,
     model_serializer,
@@ -43,13 +44,18 @@ from metagpt.const import (
     MESSAGE_ROUTE_FROM,
     MESSAGE_ROUTE_TO,
     MESSAGE_ROUTE_TO_ALL,
-    PRDS_FILE_REPO,
     SYSTEM_DESIGN_FILE_REPO,
     TASK_FILE_REPO,
 )
 from metagpt.logs import logger
 from metagpt.repo_parser import DotClassInfo
-from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, import_class
+from metagpt.utils.common import (
+    CodeParser,
+    any_to_str,
+    any_to_str_set,
+    aread,
+    import_class,
+)
 from metagpt.utils.exceptions import handle_exception
 from metagpt.utils.report import TaskReporter
 from metagpt.utils.serialize import (
@@ -157,6 +163,30 @@ class Document(BaseModel):
     def __repr__(self):
         return self.content
 
+    @classmethod
+    async def load(
+        cls, filename: Union[str, Path], project_path: Optional[Union[str, Path]] = None
+    ) -> Optional["Document"]:
+        """
+        Load a document from a file.
+
+        Args:
+            filename (Union[str, Path]): The path to the file to load.
+            project_path (Optional[Union[str, Path]], optional): The path to the project. Defaults to None.
+
+        Returns:
+            Optional[Document]: The loaded document, or None if the file does not exist.
+
+        """
+        if not filename or not Path(filename).exists():
+            return None
+        content = await aread(filename=filename)
+        doc = cls(content=content, filename=str(filename))
+        if project_path and Path(filename).is_relative_to(project_path):
+            doc.root_path = Path(filename).relative_to(project_path).parent
+            doc.filename = Path(filename).name
+        return doc
+
 
 class Documents(BaseModel):
     """A class representing a collection of documents.
@@ -360,6 +390,22 @@ class Message(BaseModel):
     def add_metadata(self, key: str, value: str):
         self.metadata[key] = value
 
+    @staticmethod
+    def create_instruct_value(kvs: Dict[str, Any], class_name: str = "") -> BaseModel:
+        """
+        Dynamically creates a Pydantic BaseModel subclass based on a given dictionary.
+
+        Parameters:
+        - data: A dictionary from which to create the BaseModel subclass.
+
+        Returns:
+        - A Pydantic BaseModel subclass instance populated with the given data.
+        """
+        if not class_name:
+            class_name = "DM" + uuid.uuid4().hex[0:8]
+        dynamic_class = create_model(class_name, **{key: (value.__class__, ...) for key, value in kvs.items()})
+        return dynamic_class.model_validate(kvs)
+
 
 class UserMessage(Message):
     """便于支持OpenAI的消息
@@ -762,22 +808,6 @@ class CodePlanAndChangeContext(BaseModel):
     design_filename: str = ""
     task_filename: str = ""
 
-    @staticmethod
-    def loads(filenames: List, **kwargs) -> CodePlanAndChangeContext:
-        ctx = CodePlanAndChangeContext(requirement=kwargs.get("requirement", ""), issue=kwargs.get("issue", ""))
-        for filename in filenames:
-            filename = Path(filename)
-            if filename.is_relative_to(PRDS_FILE_REPO):
-                ctx.prd_filename = filename.name
-                continue
-            if filename.is_relative_to(SYSTEM_DESIGN_FILE_REPO):
-                ctx.design_filename = filename.name
-                continue
-            if filename.is_relative_to(TASK_FILE_REPO):
-                ctx.task_filename = filename.name
-                continue
-        return ctx
-
 
 # mermaid class view
 class UMLClassMeta(BaseModel):

metagpt/software_company.py

@@ -68,7 +68,7 @@ def generate_repo(
     company.run_project(idea, send_to=any_to_str(ProductManager))
     asyncio.run(company.run(n_round=n_round))
 
-    return ctx.repo
+    return ctx.kwargs.get("project_path")
 
 
 @app.command("", help="Start a new project.")

metagpt/tools/libs/browser.py

@@ -1,11 +1,13 @@
 from __future__ import annotations
+
 import contextlib
+from uuid import uuid4
 
 from playwright.async_api import async_playwright
-from metagpt.utils.file import MemoryFileSystem
-from uuid import uuid4
+
 from metagpt.const import DEFAULT_WORKSPACE_ROOT
 from metagpt.tools.tool_registry import register_tool
+from metagpt.utils.file import MemoryFileSystem
 from metagpt.utils.parse_html import simplify_html
 from metagpt.utils.report import BrowserReporter
 
@@ -64,7 +66,6 @@ class Browser:
 
         # Since RAG is an optional optimization, if it fails, the simplified HTML can be used as a fallback.
         with contextlib.suppress(Exception):
-
             from metagpt.rag.engines import SimpleEngine  # avoid circular import
 
             # TODO make `from_docs` asynchronous

metagpt/utils/common.py

@@ -667,6 +667,8 @@ def role_raise_decorator(func):
 @handle_exception
 async def aread(filename: str | Path, encoding="utf-8") -> str:
     """Read file asynchronously."""
+    if not filename or not Path(filename).exists():
+        return ""
     try:
         async with aiofiles.open(str(filename), mode="r", encoding=encoding) as reader:
             content = await reader.read()
@@ -899,3 +901,51 @@ async def init_python_folder(workdir: str | Path):
         return
     async with aiofiles.open(init_filename, "a"):
         os.utime(init_filename, None)
+
+
+def get_markdown_code_block_type(filename: str) -> str:
+    if not filename:
+        return ""
+    ext = Path(filename).suffix
+    types = {
+        ".py": "python",
+        ".js": "javascript",
+        ".java": "java",
+        ".cpp": "cpp",
+        ".c": "c",
+        ".html": "html",
+        ".css": "css",
+        ".xml": "xml",
+        ".json": "json",
+        ".yaml": "yaml",
+        ".md": "markdown",
+        ".sql": "sql",
+        ".rb": "ruby",
+        ".php": "php",
+        ".sh": "bash",
+        ".swift": "swift",
+        ".go": "go",
+        ".rs": "rust",
+        ".pl": "perl",
+        ".asm": "assembly",
+        ".r": "r",
+        ".scss": "scss",
+        ".sass": "sass",
+        ".lua": "lua",
+        ".ts": "typescript",
+        ".tsx": "tsx",
+        ".jsx": "jsx",
+        ".yml": "yaml",
+        ".ini": "ini",
+        ".toml": "toml",
+        ".svg": "xml",  # SVG can often be treated as XML
+        # Add more file extensions and corresponding code block types as needed
+    }
+    return types.get(ext, "")
+
+
+def to_markdown_code_block(val: str, type_: str = "") -> str:
+    if not val:
+        return val or ""
+    val = val.replace("```", "\\`\\`\\`")
+    return f"\n```{type_}\n{val}\n```\n"

metagpt/utils/file.py

@@ -72,7 +72,6 @@ class File:
 
 
 class MemoryFileSystem(_MemoryFileSystem):
-
     @classmethod
     def _strip_protocol(cls, path):
         return super()._strip_protocol(str(path))

metagpt/utils/git_repository.py

@@ -156,6 +156,8 @@ class GitRepository:
         :param local_path: The local path to check.
         :return: True if the directory is a Git repository, False otherwise.
         """
+        if not local_path:
+            return False
         git_dir = Path(local_path) / ".git"
         if git_dir.exists() and is_git_dir(git_dir):
             return True

metagpt/utils/parse_html.py

@@ -4,11 +4,10 @@ from __future__ import annotations
 from typing import Generator, Optional
 from urllib.parse import urljoin, urlparse
 
+import htmlmin
 from bs4 import BeautifulSoup
 from pydantic import BaseModel, PrivateAttr
 
-import htmlmin
-
 
 class WebPage(BaseModel):
     inner_text: str

metagpt/utils/project_repo.py

@@ -140,10 +140,11 @@ class ProjectRepo(FileRepository):
         return bool(code_files)
 
     def with_src_path(self, path: str | Path) -> ProjectRepo:
-        try:
-            self._srcs_path = Path(path).relative_to(self.workdir)
-        except ValueError:
-            self._srcs_path = Path(path)
+        path = Path(path)
+        if path.is_relative_to(self.workdir):
+            self._srcs_path = path.relative_to(self.workdir)
+        else:
+            self._srcs_path = path
         return self
 
     @property

tests/conftest.py

@@ -12,7 +12,6 @@ import logging
 import os
 import re
 import uuid
-from pathlib import Path
 from typing import Callable
 
 import aiohttp.web
@@ -23,7 +22,6 @@ from metagpt.context import Context as MetagptContext
 from metagpt.llm import LLM
 from metagpt.logs import logger
 from metagpt.utils.git_repository import GitRepository
-from metagpt.utils.project_repo import ProjectRepo
 from tests.mock.mock_aiohttp import MockAioResponse
 from tests.mock.mock_curl_cffi import MockCurlCffiResponse
 from tests.mock.mock_httplib2 import MockHttplib2Response
@@ -149,13 +147,14 @@ def loguru_caplog(caplog):
 @pytest.fixture(scope="function")
 def context(request):
     ctx = MetagptContext()
-    ctx.git_repo = GitRepository(local_path=DEFAULT_WORKSPACE_ROOT / f"unittest/{uuid.uuid4().hex}")
-    ctx.repo = ProjectRepo(ctx.git_repo)
+    repo = GitRepository(local_path=DEFAULT_WORKSPACE_ROOT / f"unittest/{uuid.uuid4().hex}")
+    ctx.config.project_path = str(repo.workdir)
 
     # Destroy git repo at the end of the test session.
     def fin():
-        if ctx.git_repo:
-            ctx.git_repo.delete_repository()
+        if ctx.config.project_path:
+            git_repo = GitRepository(ctx.config.project_path)
+            git_repo.delete_repository()
 
     # Register the function for destroying the environment.
     request.addfinalizer(fin)
@@ -279,6 +278,6 @@ def mermaid_mocker(aiohttp_mocker, mermaid_rsp_cache):
 @pytest.fixture
 def git_dir():
     """Fixture to get the unittest directory."""
-    git_dir = Path(__file__).parent / f"unittest/{uuid.uuid4().hex}"
+    git_dir = DEFAULT_WORKSPACE_ROOT / f"unittest/{uuid.uuid4().hex}"
     git_dir.mkdir(parents=True, exist_ok=True)
     return git_dir

tests/metagpt/actions/test_action_node.py

@@ -303,5 +303,4 @@ def test_action_node_from_pydantic_and_print_everything():
 
 
 if __name__ == "__main__":
-    test_create_model_class()
-    test_create_model_class_with_mapping()
+    pytest.main([__file__, "-s"])

tests/metagpt/actions/test_design_api.py

@@ -6,37 +6,104 @@
 @File    : test_design_api.py
 @Modifiled By: mashenquan, 2023-12-6. According to RFC 135
 """
+from pathlib import Path
+
 import pytest
 
 from metagpt.actions.design_api import WriteDesign
-from metagpt.llm import LLM
+from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT
 from metagpt.logs import logger
-from metagpt.schema import Message
+from metagpt.schema import AIMessage, Message
+from metagpt.utils.project_repo import ProjectRepo
 from tests.data.incremental_dev_project.mock import DESIGN_SAMPLE, REFINED_PRD_JSON
 
 
 @pytest.mark.asyncio
-async def test_design_api(context):
-    inputs = ["我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。"]  # PRD_SAMPLE
-    for prd in inputs:
-        await context.repo.docs.prd.save(filename="new_prd.txt", content=prd)
+async def test_design(context):
+    # Mock new design env
+    prd = "我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。"
+    context.kwargs.project_path = context.config.project_path
+    context.kwargs.inc = False
+    filename = "prd.txt"
+    repo = ProjectRepo(context.kwargs.project_path)
+    await repo.docs.prd.save(filename=filename, content=prd)
+    kvs = {
+        "project_path": str(context.kwargs.project_path),
+        "changed_prd_filenames": [str(repo.docs.prd.workdir / filename)],
+    }
+    instruct_content = AIMessage.create_instruct_value(kvs=kvs, class_name="WritePRDOutput")
 
-        design_api = WriteDesign(context=context)
-
-        result = await design_api.run(Message(content=prd, instruct_content=None))
-        logger.info(result)
-
-        assert result
-
-
-@pytest.mark.asyncio
-async def test_refined_design_api(context):
-    await context.repo.docs.prd.save(filename="1.txt", content=str(REFINED_PRD_JSON))
-    await context.repo.docs.system_design.save(filename="1.txt", content=DESIGN_SAMPLE)
-
-    design_api = WriteDesign(context=context, llm=LLM())
-
-    result = await design_api.run(Message(content="", instruct_content=None))
+    design_api = WriteDesign(context=context)
+    result = await design_api.run([Message(content=prd, instruct_content=instruct_content)])
     logger.info(result)
-
     assert result
+    assert isinstance(result, AIMessage)
+    assert result.instruct_content
+    assert repo.docs.system_design.changed_files
+
+    # Mock incremental design env
+    context.kwargs.inc = True
+    await repo.docs.prd.save(filename=filename, content=str(REFINED_PRD_JSON))
+    await repo.docs.system_design.save(filename=filename, content=DESIGN_SAMPLE)
+
+    result = await design_api.run([Message(content="", instruct_content=instruct_content)])
+    logger.info(result)
+    assert result
+    assert isinstance(result, AIMessage)
+    assert result.instruct_content
+    assert repo.docs.system_design.changed_files
+
+
+@pytest.mark.parametrize(
+    ("user_requirement", "prd_filename", "legacy_design_filename"),
+    [
+        ("我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", None, None),
+        ("write 2048 game", str(METAGPT_ROOT / "tests/data/prd.json"), None),
+        (
+            "write 2048 game",
+            str(METAGPT_ROOT / "tests/data/prd.json"),
+            str(METAGPT_ROOT / "tests/data/system_design.json"),
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_design_api(context, user_requirement, prd_filename, legacy_design_filename):
+    action = WriteDesign()
+    result = await action.run(
+        user_requirement=user_requirement, prd_filename=prd_filename, legacy_design_filename=legacy_design_filename
+    )
+    assert isinstance(result, AIMessage)
+    assert result.content
+    assert str(DEFAULT_WORKSPACE_ROOT) in result.content
+
+
+@pytest.mark.parametrize(
+    ("user_requirement", "prd_filename", "legacy_design_filename"),
+    [
+        ("我们需要一个音乐播放器，它应该有播放、暂停、上一曲、下一曲等功能。", None, None),
+        ("write 2048 game", str(METAGPT_ROOT / "tests/data/prd.json"), None),
+        (
+            "write 2048 game",
+            str(METAGPT_ROOT / "tests/data/prd.json"),
+            str(METAGPT_ROOT / "tests/data/system_design.json"),
+        ),
+    ],
+)
+@pytest.mark.asyncio
+async def test_design_api_dir(context, user_requirement, prd_filename, legacy_design_filename):
+    action = WriteDesign()
+    result = await action.run(
+        user_requirement=user_requirement,
+        prd_filename=prd_filename,
+        legacy_design_filename=legacy_design_filename,
+        output_pathname=str(Path(context.config.project_path) / "1.txt"),
+    )
+    assert isinstance(result, AIMessage)
+    assert result.content
+    assert str(context.config.project_path) in result.content
+    assert result.instruct_content
+    assert result.instruct_content.changed_system_design_filenames
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-s"])

tests/metagpt/actions/test_project_management.py

@@ -5,13 +5,15 @@
 @Author  : alexanderwu
 @File    : test_project_management.py
 """
+import json
 
 import pytest
 
 from metagpt.actions.project_management import WriteTasks
-from metagpt.llm import LLM
+from metagpt.const import METAGPT_ROOT
 from metagpt.logs import logger
-from metagpt.schema import Message
+from metagpt.schema import AIMessage, Message
+from metagpt.utils.project_repo import ProjectRepo
 from tests.data.incremental_dev_project.mock import (
     REFINED_DESIGN_JSON,
     REFINED_PRD_JSON,
@@ -22,29 +24,46 @@ from tests.metagpt.actions.mock_json import DESIGN, PRD
 
 @pytest.mark.asyncio
 async def test_task(context):
-    await context.repo.docs.prd.save("1.txt", content=str(PRD))
-    await context.repo.docs.system_design.save("1.txt", content=str(DESIGN))
-    logger.info(context.git_repo)
+    # Mock write tasks env
+    context.kwargs.project_path = context.config.project_path
+    context.kwargs.inc = False
+    repo = ProjectRepo(context.kwargs.project_path)
+    filename = "1.txt"
+    await repo.docs.prd.save(filename=filename, content=str(PRD))
+    await repo.docs.system_design.save(filename=filename, content=str(DESIGN))
+    kvs = {
+        "project_path": context.kwargs.project_path,
+        "changed_system_design_filenames": [str(repo.docs.system_design.workdir / filename)],
+    }
+    instruct_content = AIMessage.create_instruct_value(kvs=kvs, class_name="WriteDesignOutput")
 
     action = WriteTasks(context=context)
-
-    result = await action.run(Message(content="", instruct_content=None))
+    result = await action.run([Message(content="", instruct_content=instruct_content)])
     logger.info(result)
-
     assert result
+    assert result.instruct_content.changed_task_filenames
+
+    # Mock incremental env
+    context.kwargs.inc = True
+    await repo.docs.prd.save(filename=filename, content=str(REFINED_PRD_JSON))
+    await repo.docs.system_design.save(filename=filename, content=str(REFINED_DESIGN_JSON))
+    await repo.docs.task.save(filename=filename, content=TASK_SAMPLE)
+
+    result = await action.run([Message(content="", instruct_content=instruct_content)])
+    logger.info(result)
+    assert result
+    assert result.instruct_content.changed_task_filenames
 
 
 @pytest.mark.asyncio
-async def test_refined_task(context):
-    await context.repo.docs.prd.save("2.txt", content=str(REFINED_PRD_JSON))
-    await context.repo.docs.system_design.save("2.txt", content=str(REFINED_DESIGN_JSON))
-    await context.repo.docs.task.save("2.txt", content=TASK_SAMPLE)
-
-    logger.info(context.git_repo)
-
-    action = WriteTasks(context=context, llm=LLM())
-
-    result = await action.run(Message(content="", instruct_content=None))
-    logger.info(result)
-
+async def test_task_api(context):
+    action = WriteTasks()
+    result = await action.run(design_filename=str(METAGPT_ROOT / "tests/data/system_design.json"))
     assert result
+    assert result.content
+    m = json.loads(result.content)
+    assert m
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-s"])

tests/metagpt/actions/test_write_code.py

@@ -26,12 +26,7 @@ from tests.metagpt.actions.mock_markdown import TASKS_2, WRITE_CODE_PROMPT_SAMPL
 
 def setup_inc_workdir(context, inc: bool = False):
     """setup incremental workdir for testing"""
-    context.src_workspace = context.git_repo.workdir / "src"
-    if inc:
-        context.config.inc = inc
-        context.repo.old_workspace = context.repo.git_repo.workdir / "old"
-        context.config.project_path = "old"
-
+    context.config.inc = inc
     return context
 
 

tests/metagpt/actions/test_write_prd.py

@@ -6,25 +6,26 @@
 @File    : test_write_prd.py
 @Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, replace `handle` with `run`.
 """
+import uuid
+from pathlib import Path
 
 import pytest
 
 from metagpt.actions import UserRequirement, WritePRD
-from metagpt.const import REQUIREMENT_FILENAME
+from metagpt.const import DEFAULT_WORKSPACE_ROOT, REQUIREMENT_FILENAME
 from metagpt.logs import logger
 from metagpt.roles.product_manager import ProductManager
 from metagpt.roles.role import RoleReactMode
-from metagpt.schema import Message
+from metagpt.schema import AIMessage, Message
 from metagpt.utils.common import any_to_str
-from tests.data.incremental_dev_project.mock import NEW_REQUIREMENT_SAMPLE, PRD_SAMPLE
-from tests.metagpt.actions.test_write_code import setup_inc_workdir
+from metagpt.utils.project_repo import ProjectRepo
+from tests.data.incremental_dev_project.mock import NEW_REQUIREMENT_SAMPLE
 
 
 @pytest.mark.asyncio
 async def test_write_prd(new_filename, context):
     product_manager = ProductManager(context=context)
     requirements = "开发一个基于大语言模型与私有知识库的搜索引擎，希望可以基于大语言模型进行搜索总结"
-    await context.repo.docs.save(filename=REQUIREMENT_FILENAME, content=requirements)
     product_manager.rc.react_mode = RoleReactMode.BY_ORDER
     prd = await product_manager.run(Message(content=requirements, cause_by=UserRequirement))
     assert prd.cause_by == any_to_str(WritePRD)
@@ -34,38 +35,39 @@ async def test_write_prd(new_filename, context):
     # Assert the prd is not None or empty
     assert prd is not None
     assert prd.content != ""
-    assert product_manager.context.repo.docs.prd.changed_files
+    repo = ProjectRepo(context.kwargs.project_path)
+    assert repo.docs.prd.changed_files
+    repo.git_repo.archive()
 
-
-@pytest.mark.asyncio
-async def test_write_prd_inc(new_filename, context, git_dir):
-    context = setup_inc_workdir(context, inc=True)
-    await context.repo.docs.prd.save("1.txt", PRD_SAMPLE)
-    await context.repo.docs.save(filename=REQUIREMENT_FILENAME, content=NEW_REQUIREMENT_SAMPLE)
+    # Mock incremental requirement
+    context.config.inc = True
+    context.config.project_path = context.kwargs.project_path
+    repo = ProjectRepo(context.config.project_path)
+    await repo.docs.save(filename=REQUIREMENT_FILENAME, content=NEW_REQUIREMENT_SAMPLE)
 
     action = WritePRD(context=context)
-    prd = await action.run(Message(content=NEW_REQUIREMENT_SAMPLE, instruct_content=None))
+    prd = await action.run([Message(content=NEW_REQUIREMENT_SAMPLE, instruct_content=None)])
     logger.info(NEW_REQUIREMENT_SAMPLE)
     logger.info(prd)
 
     # Assert the prd is not None or empty
     assert prd is not None
     assert prd.content != ""
-    assert "Refined Requirements" in prd.content
+    assert repo.git_repo.changed_files
 
 
 @pytest.mark.asyncio
 async def test_fix_debug(new_filename, context, git_dir):
-    context.src_workspace = context.git_repo.workdir / context.git_repo.workdir.name
+    # Mock legacy project
+    context.kwargs.project_path = str(git_dir)
+    repo = ProjectRepo(context.kwargs.project_path)
+    repo.with_src_path(git_dir.name)
+    await repo.srcs.save(filename="main.py", content='if __name__ == "__main__":\nmain()')
+    requirements = "ValueError: undefined variable `st`."
+    await repo.docs.save(filename=REQUIREMENT_FILENAME, content=requirements)
 
-    await context.repo.with_src_path(context.src_workspace).srcs.save(
-        filename="main.py", content='if __name__ == "__main__":\nmain()'
-    )
-    requirements = "Please fix the bug in the code."
-    await context.repo.docs.save(filename=REQUIREMENT_FILENAME, content=requirements)
     action = WritePRD(context=context)
-
-    prd = await action.run(Message(content=requirements, instruct_content=None))
+    prd = await action.run([Message(content=requirements, instruct_content=None)])
     logger.info(prd)
 
     # Assert the prd is not None or empty
@@ -73,5 +75,40 @@ async def test_fix_debug(new_filename, context, git_dir):
     assert prd.content != ""
 
 
+@pytest.mark.asyncio
+async def test_write_prd_api(context):
+    action = WritePRD()
+    result = await action.run(user_requirement="write a snake game.")
+    assert isinstance(result, AIMessage)
+    assert result.content
+    assert str(DEFAULT_WORKSPACE_ROOT) in result.content
+
+    result = await action.run(
+        user_requirement="write a snake game.",
+        output_pathname=str(Path(context.config.project_path) / f"{uuid.uuid4().hex}.json"),
+    )
+    assert isinstance(result, AIMessage)
+    assert result.content
+    assert result.instruct_content
+    assert str(context.config.project_path) in result.content
+
+    legacy_prd_filename = result.instruct_content.changed_prd_filenames[-1]
+
+    result = await action.run(user_requirement="Add moving enemy.", legacy_prd_filename=legacy_prd_filename)
+    assert isinstance(result, AIMessage)
+    assert result.content
+    assert str(DEFAULT_WORKSPACE_ROOT) in result.content
+
+    result = await action.run(
+        user_requirement="Add moving enemy.",
+        output_pathname=str(Path(context.config.project_path) / f"{uuid.uuid4().hex}.json"),
+        legacy_prd_filename=legacy_prd_filename,
+    )
+    assert isinstance(result, AIMessage)
+    assert result.content
+    assert result.instruct_content
+    assert str(context.config.project_path) in result.content
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-s"])

tests/metagpt/test_schema.py

@@ -392,5 +392,11 @@ async def test_parse_resources(context, content: str, key_descriptions):
         assert k in result
 
 
+@pytest.mark.parametrize(("name", "value"), [("c1", {"age": 10, "name": "Alice"}), ("", {"path": __file__})])
+def test_create_instruct_value(name, value):
+    obj = Message.create_instruct_value(kvs=value, class_name=name)
+    assert obj.model_dump() == value
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-s"])