apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-02 20:32:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: new/inc/patch pass

Browse source
This commit is contained in:
莘权 马 2024-05-18 14:36:22 +08:00
parent 5c416a1f31
commit ee0b9d2039
21 changed files with 512 additions and 237 deletions
Download patch file Download diff file Expand all files Collapse all files

7
metagpt/actions/action.py
View file

@ -22,7 +22,6 @@ from metagpt.schema import (
SerializationMixin,
TestingContext,
)
from metagpt.utils.project_repo import ProjectRepo
class Action(SerializationMixin, ContextMixin, BaseModel):
@ -36,12 +35,6 @@ class Action(SerializationMixin, ContextMixin, BaseModel):
desc: str = "" # for skill manager
node: ActionNode = Field(default=None, exclude=True)
@property
def repo(self) -> ProjectRepo:
if not self.context.repo:
self.context.repo = ProjectRepo(self.context.git_repo)
return self.context.repo
@property
def prompt_schema(self):
return self.config.prompt_schema

10
metagpt/actions/debug_error.py
View file

@ -9,13 +9,15 @@
2. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
"""
import re
from typing import Optional
from pydantic import Field
from pydantic import BaseModel, Field
from metagpt.actions.action import Action
from metagpt.logs import logger
from metagpt.schema import RunCodeContext, RunCodeResult
from metagpt.utils.common import CodeParser
from metagpt.utils.project_repo import ProjectRepo
PROMPT_TEMPLATE = """
NOTICE
@ -47,6 +49,8 @@ Now you should start rewriting the code:
class DebugError(Action):
i_context: RunCodeContext = Field(default_factory=RunCodeContext)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
async def run(self, *args, **kwargs) -> str:
output_doc = await self.repo.test_outputs.get(filename=self.i_context.output_filename)
@ -59,9 +63,7 @@ class DebugError(Action):
return ""
logger.info(f"Debug and rewrite {self.i_context.test_filename}")
code_doc = await self.repo.with_src_path(self.context.src_workspace).srcs.get(
filename=self.i_context.code_filename
)
code_doc = await self.repo.srcs.get(filename=self.i_context.code_filename)
if not code_doc:
return ""
test_doc = await self.repo.tests.get(filename=self.i_context.test_filename)

34
metagpt/actions/design_api.py
View file

@ -13,6 +13,8 @@ import json
from pathlib import Path
from typing import Optional
from pydantic import BaseModel, Field
from metagpt.actions import Action
from metagpt.actions.design_api_an import (
DATA_STRUCTURES_AND_INTERFACES,
@ -26,6 +28,7 @@ from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import AIMessage, Document, Documents, Message
from metagpt.utils.mermaid import mermaid_to_file
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import DocsReporter, GalleryReporter
NEW_REQ_TEMPLATE = """
@ -45,21 +48,25 @@ class WriteDesign(Action):
"data structures, library tables, processes, and paths. Please provide your design, feedback "
"clearly and in detail."
)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
async def run(self, with_messages: Message, schema: str = None):
# Use `git status` to identify which PRD documents have been modified in the `docs/prd` directory.
changed_prds = self.repo.docs.prd.changed_files
# Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone
# changes.
changed_system_designs = self.repo.docs.system_design.changed_files
self.input_args = with_messages[0].instruct_content
self.repo = ProjectRepo(self.input_args.project_path)
changed_prds = self.input_args.changed_prd_filenames
changed_system_designs = [
str(self.repo.docs.system_design.workdir / i)
for i in list(self.repo.docs.system_design.changed_files.keys())
]
# For those PRDs and design documents that have undergone changes, regenerate the design content.
changed_files = Documents()
for filename in changed_prds.keys():
for filename in changed_prds:
doc = await self._update_system_design(filename=filename)
changed_files.docs[filename] = doc
for filename in changed_system_designs.keys():
for filename in changed_system_designs:
if filename in changed_files.docs:
continue
doc = await self._update_system_design(filename=filename)
@ -68,6 +75,11 @@ class WriteDesign(Action):
logger.info("Nothing has changed.")
# Wait until all files under `docs/system_designs/` are processed before sending the publish message,
# leaving room for global optimization in subsequent steps.
kvs = self.input_args.model_dump()
kvs["changed_system_design_filenames"] = [
str(self.repo.docs.system_design.workdir / i)
for i in list(self.repo.docs.system_design.changed_files.keys())
]
return AIMessage(
content="Designing is complete. "
+ "\n".join(
@ -75,6 +87,7 @@ class WriteDesign(Action):
+ list(self.repo.resources.data_api_design.changed_files.keys())
+ list(self.repo.resources.seq_flow.changed_files.keys())
),
instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteDesignOutput"),
cause_by=self,
)
@ -89,14 +102,15 @@ class WriteDesign(Action):
return system_design_doc
async def _update_system_design(self, filename) -> Document:
prd = await self.repo.docs.prd.get(filename)
old_system_design_doc = await self.repo.docs.system_design.get(filename)
root_relative_path = Path(filename).relative_to(self.repo.workdir)
prd = await Document.load(filename=filename, project_path=self.repo.workdir)
old_system_design_doc = await self.repo.docs.system_design.get(root_relative_path.name)
async with DocsReporter(enable_llm_stream=True) as reporter:
await reporter.async_report({"type": "design"}, "meta")
if not old_system_design_doc:
system_design = await self._new_system_design(context=prd.content)
doc = await self.repo.docs.system_design.save(
filename=filename,
filename=prd.filename,
content=system_design.instruct_content.model_dump_json(),
dependencies={prd.root_relative_path},
)

22
metagpt/actions/prepare_documents.py
View file

@ -17,6 +17,7 @@ from metagpt.logs import logger
from metagpt.schema import AIMessage
from metagpt.utils.common import any_to_str
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.project_repo import ProjectRepo
class PrepareDocuments(Action):
@ -36,7 +37,7 @@ class PrepareDocuments(Action):
def config(self):
return self.context.config
def _init_repo(self):
def _init_repo(self) -> ProjectRepo:
"""Initialize the Git environment."""
if not self.config.project_path:
name = self.config.project_name or FileRepository.new_filename()
@ -47,6 +48,7 @@ class PrepareDocuments(Action):
shutil.rmtree(path)
self.config.project_path = path
self.context.set_repo_dir(path)
return ProjectRepo(path)
async def run(self, with_messages, **kwargs):
"""Create and initialize the workspace folder, initialize the Git environment."""
@ -67,10 +69,22 @@ class PrepareDocuments(Action):
max_auto_summarize_code=0,
)
self._init_repo()
repo = self._init_repo()
# Write the newly added requirements from the main parameter idea to `docs/requirement.txt`.
doc = await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
await repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content)
# Send a Message notification to the WritePRD action, instructing it to process requirements using
# `docs/requirement.txt` and `docs/prd/`.
return AIMessage(content="", instruct_content=doc, cause_by=self, send_to=self.send_to)
return AIMessage(
content="",
instruct_content=AIMessage.create_instruct_value(
kvs={
"project_path": str(repo.workdir),
"requirements_filename": str(repo.docs.workdir / REQUIREMENT_FILENAME),
"prd_filenames": [str(repo.docs.prd.workdir / i) for i in repo.docs.prd.all_files],
},
class_name="PrepareDocumentsOutput",
),
cause_by=self,
send_to=self.send_to,
)

25
metagpt/actions/project_management.py
View file

@ -11,13 +11,17 @@
"""
import json
from pathlib import Path
from typing import Optional
from pydantic import BaseModel, Field
from metagpt.actions.action import Action
from metagpt.actions.project_management_an import PM_NODE, REFINED_PM_NODE
from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME
from metagpt.logs import logger
from metagpt.schema import AIMessage, Document, Documents
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import DocsReporter
NEW_REQ_TEMPLATE = """
@ -32,10 +36,14 @@ NEW_REQ_TEMPLATE = """
class WriteTasks(Action):
name: str = "CreateTasks"
i_context: Optional[str] = None
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
async def run(self, with_messages):
changed_system_designs = self.repo.docs.system_design.changed_files
changed_tasks = self.repo.docs.task.changed_files
self.input_args = with_messages[0].instruct_content
self.repo = ProjectRepo(self.input_args.project_path)
changed_system_designs = self.input_args.changed_system_design_filenames
changed_tasks = [str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())]
change_files = Documents()
# Rewrite the system designs that have undergone changes based on the git head diff under
# `docs/system_designs/`.
@ -54,6 +62,11 @@ class WriteTasks(Action):
logger.info("Nothing has changed.")
# Wait until all files under `docs/tasks/` are processed before sending the publish_message, leaving room for
# global optimization in subsequent steps.
kvs = self.input_args.model_dump()
kvs["changed_task_filenames"] = [
str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())
]
kvs["python_package_dependency_filename"] = str(self.repo.workdir / PACKAGE_REQUIREMENTS_FILENAME)
return AIMessage(
content="WBS is completed. "
+ "\n".join(
@ -61,12 +74,14 @@ class WriteTasks(Action):
+ list(self.repo.docs.task.changed_files.keys())
+ list(self.repo.resources.api_spec_and_task.changed_files.keys())
),
instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTaskOutput"),
cause_by=self,
)
async def _update_tasks(self, filename):
system_design_doc = await self.repo.docs.system_design.get(filename)
task_doc = await self.repo.docs.task.get(filename)
root_relative_path = Path(filename).relative_to(self.repo.workdir)
system_design_doc = await Document.load(filename=filename, project_path=self.repo.workdir)
task_doc = await self.repo.docs.task.get(root_relative_path.name)
async with DocsReporter(enable_llm_stream=True) as reporter:
await reporter.async_report({"type": "task"}, "meta")
if task_doc:
@ -75,7 +90,7 @@ class WriteTasks(Action):
else:
rsp = await self._run_new_tasks(context=system_design_doc.content)
task_doc = await self.repo.docs.task.save(
filename=filename,
filename=system_design_doc.filename,
content=rsp.instruct_content.model_dump_json(),
dependencies={system_design_doc.root_relative_path},
)

12
metagpt/actions/summarize_code.py
View file

@ -6,13 +6,16 @@
@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode.
"""
from pathlib import Path
from typing import Optional
from pydantic import Field
from pydantic import BaseModel, Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action import Action
from metagpt.logs import logger
from metagpt.schema import CodeSummarizeContext
from metagpt.utils.common import get_markdown_code_block_type
from metagpt.utils.project_repo import ProjectRepo
PROMPT_TEMPLATE = """
NOTICE
@ -90,6 +93,8 @@ flowchart TB
class SummarizeCode(Action):
name: str = "SummarizeCode"
i_context: CodeSummarizeContext = Field(default_factory=CodeSummarizeContext)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
@retry(stop=stop_after_attempt(2), wait=wait_random_exponential(min=1, max=60))
async def summarize_code(self, prompt):
@ -101,11 +106,10 @@ class SummarizeCode(Action):
design_doc = await self.repo.docs.system_design.get(filename=design_pathname.name)
task_pathname = Path(self.i_context.task_filename)
task_doc = await self.repo.docs.task.get(filename=task_pathname.name)
src_file_repo = self.repo.with_src_path(self.context.src_workspace).srcs
code_blocks = []
for filename in self.i_context.codes_filenames:
code_doc = await src_file_repo.get(filename)
code_block = f"```python\n{code_doc.content}\n```\n-----"
code_doc = await self.repo.srcs.get(filename)
code_block = f"```{get_markdown_code_block_type(filename)}\n{code_doc.content}\n```\n---\n"
code_blocks.append(code_block)
format_example = FORMAT_EXAMPLE
prompt = PROMPT_TEMPLATE.format(

72
metagpt/actions/write_code.py
View file

@ -16,17 +16,18 @@
"""
import json
from pathlib import Path
from typing import Optional
from pydantic import Field
from pydantic import BaseModel, Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action import Action
from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
from metagpt.actions.write_code_plan_and_change_an import REFINED_TEMPLATE
from metagpt.const import BUGFIX_FILENAME, REQUIREMENT_FILENAME
from metagpt.logs import logger
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser
from metagpt.utils.common import CodeParser, get_markdown_code_block_type
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import EditorReporter
@ -44,9 +45,7 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
{task}
## Legacy Code
```Code
{code}
```
## Debug logs
```text
@ -61,14 +60,14 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
```
# Format example
## Code: {filename}
## Code: {demo_filename}.py
```python
## {filename}
## {demo_filename}.py
...
```
## Code: {filename}
## Code: {demo_filename}.js
```javascript
// {filename}
// {demo_filename}.js
...
```
@ -89,6 +88,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
class WriteCode(Action):
name: str = "WriteCode"
i_context: Document = Field(default_factory=Document)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
async def write_code(self, prompt) -> str:
@ -97,10 +98,16 @@ class WriteCode(Action):
return code
async def run(self, *args, **kwargs) -> CodingContext:
bug_feedback = await self.repo.docs.get(filename=BUGFIX_FILENAME)
bug_feedback = None
if self.input_args and hasattr(self.input_args, "issue_filename"):
bug_feedback = await Document.load(self.input_args.issue_filename)
coding_context = CodingContext.loads(self.i_context.content)
if not coding_context.code_plan_and_change_doc:
coding_context.code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(
filename=coding_context.task_doc.filename
)
test_doc = await self.repo.test_outputs.get(filename="test_" + coding_context.filename + ".json")
requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
requirement_doc = await Document.load(self.input_args.requirements_filename)
summary_doc = None
if coding_context.design_doc and coding_context.design_doc.filename:
summary_doc = await self.repo.docs.code_summary.get(filename=coding_context.design_doc.filename)
@ -109,29 +116,28 @@ class WriteCode(Action):
test_detail = RunCodeResult.loads(test_doc.content)
logs = test_detail.stderr
if bug_feedback:
code_context = coding_context.code_doc.content
elif self.config.inc:
if self.config.inc or bug_feedback:
code_context = await self.get_codes(
coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo, use_inc=True
)
else:
code_context = await self.get_codes(
coding_context.task_doc,
exclude=self.i_context.filename,
project_repo=self.repo.with_src_path(self.context.src_workspace),
coding_context.task_doc, exclude=self.i_context.filename, project_repo=self.repo
)
if self.config.inc:
prompt = REFINED_TEMPLATE.format(
user_requirement=requirement_doc.content if requirement_doc else "",
code_plan_and_change=str(coding_context.code_plan_and_change_doc),
code_plan_and_change=coding_context.code_plan_and_change_doc.content
if coding_context.code_plan_and_change_doc
else "",
design=coding_context.design_doc.content if coding_context.design_doc else "",
task=coding_context.task_doc.content if coding_context.task_doc else "",
code=code_context,
logs=logs,
feedback=bug_feedback.content if bug_feedback else "",
filename=self.i_context.filename,
demo_filename=Path(self.i_context.filename).stem,
summary_log=summary_doc.content if summary_doc else "",
)
else:
@ -142,6 +148,7 @@ class WriteCode(Action):
logs=logs,
feedback=bug_feedback.content if bug_feedback else "",
filename=self.i_context.filename,
demo_filename=Path(self.i_context.filename).stem,
summary_log=summary_doc.content if summary_doc else "",
)
logger.info(f"Writing {coding_context.filename}..")
@ -150,8 +157,9 @@ class WriteCode(Action):
code = await self.write_code(prompt)
if not coding_context.code_doc:
# avoid root_path pydantic ValidationError if use WriteCode alone
root_path = self.context.src_workspace if self.context.src_workspace else ""
coding_context.code_doc = Document(filename=coding_context.filename, root_path=str(root_path))
coding_context.code_doc = Document(
filename=coding_context.filename, root_path=str(self.repo.src_relative_path)
)
coding_context.code_doc.content = code
await reporter.async_report(self.repo.workdir / coding_context.code_doc.root_relative_path, "path")
return coding_context
@ -178,35 +186,32 @@ class WriteCode(Action):
code_filenames = m.get(TASK_LIST.key, []) if not use_inc else m.get(REFINED_TASK_LIST.key, [])
codes = []
src_file_repo = project_repo.srcs
# Incremental development scenario
if use_inc:
src_files = src_file_repo.all_files
# Get the old workspace contained the old codes and old workspace are created in previous CodePlanAndChange
old_file_repo = project_repo.git_repo.new_file_repository(relative_path=project_repo.old_workspace)
old_files = old_file_repo.all_files
# Get the union of the files in the src and old workspaces
union_files_list = list(set(src_files) | set(old_files))
for filename in union_files_list:
for filename in src_file_repo.all_files:
code_block_type = get_markdown_code_block_type(filename)
# Exclude the current file from the all code snippets
if filename == exclude:
# If the file is in the old workspace, use the old code
# Exclude unnecessary code to maintain a clean and focused main.py file, ensuring only relevant and
# essential functionality is included for the project’s requirements
if filename in old_files and filename != "main.py":
if filename != "main.py":
# Use old code
doc = await old_file_repo.get(filename=filename)
doc = await src_file_repo.get(filename=filename)
# If the file is in the src workspace, skip it
else:
continue
codes.insert(0, f"-----Now, {filename} to be rewritten\n```{doc.content}```\n=====")
codes.insert(
0, f"### The name of file to rewrite: `{filename}`\n```{code_block_type}\n{doc.content}```\n"
)
logger.info(f"Prepare to rewrite `{filename}`")
# The code snippets are generated from the src workspace
else:
doc = await src_file_repo.get(filename=filename)
# If the file does not exist in the src workspace, skip it
if not doc:
continue
codes.append(f"----- {filename}\n```{doc.content}```")
codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n")
# Normal scenario
else:
@ -217,6 +222,7 @@ class WriteCode(Action):
doc = await src_file_repo.get(filename=filename)
if not doc:
continue
codes.append(f"----- {filename}\n```{doc.content}```")
code_block_type = get_markdown_code_block_type(filename)
codes.append(f"### File Name: `{filename}`\n```{code_block_type}\n{doc.content}```\n\n")
return "\n".join(codes)

37
metagpt/actions/write_code_plan_and_change_an.py
View file

@ -5,15 +5,16 @@
@Author : mannaandpoem
@File : write_code_plan_and_change_an.py
"""
import os
from typing import List
from typing import List, Optional
from pydantic import Field
from pydantic import BaseModel, Field
from metagpt.actions.action import Action
from metagpt.actions.action_node import ActionNode
from metagpt.logs import logger
from metagpt.schema import CodePlanAndChangeContext
from metagpt.schema import CodePlanAndChangeContext, Document
from metagpt.utils.common import get_markdown_code_block_type
from metagpt.utils.project_repo import ProjectRepo
DEVELOPMENT_PLAN = ActionNode(
key="Development Plan",
@ -162,9 +163,8 @@ Role: You are a professional engineer; The main goal is to complete incremental
{task}
## Legacy Code
```Code
{code}
```
## Debug logs
```text
@ -179,14 +179,14 @@ Role: You are a professional engineer; The main goal is to complete incremental
```
# Format example
## Code: {filename}
## Code: {demo_filename}.py
```python
## {filename}
## {demo_filename}.py
...
```
## Code: {filename}
## Code: {demo_filename}.js
```javascript
// {filename}
// {demo_filename}.js
...
```
@ -211,13 +211,15 @@ WRITE_CODE_PLAN_AND_CHANGE_NODE = ActionNode.from_children("WriteCodePlanAndChan
class WriteCodePlanAndChange(Action):
name: str = "WriteCodePlanAndChange"
i_context: CodePlanAndChangeContext = Field(default_factory=CodePlanAndChangeContext)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
async def run(self, *args, **kwargs):
self.llm.system_prompt = "You are a professional software engineer, your primary responsibility is to "
"meticulously craft comprehensive incremental development plan and deliver detailed incremental change"
prd_doc = await self.repo.docs.prd.get(filename=self.i_context.prd_filename)
design_doc = await self.repo.docs.system_design.get(filename=self.i_context.design_filename)
task_doc = await self.repo.docs.task.get(filename=self.i_context.task_filename)
prd_doc = await Document.load(filename=self.i_context.prd_filename)
design_doc = await Document.load(filename=self.i_context.design_filename)
task_doc = await Document.load(filename=self.i_context.task_filename)
context = CODE_PLAN_AND_CHANGE_CONTEXT.format(
requirement=f"```text\n{self.i_context.requirement}\n```",
issue=f"```text\n{self.i_context.issue}\n```",
@ -230,8 +232,9 @@ class WriteCodePlanAndChange(Action):
return await WRITE_CODE_PLAN_AND_CHANGE_NODE.fill(context=context, llm=self.llm, schema="json")
async def get_old_codes(self) -> str:
self.repo.old_workspace = self.repo.git_repo.workdir / os.path.basename(self.config.project_path)
old_file_repo = self.repo.git_repo.new_file_repository(relative_path=self.repo.old_workspace)
old_codes = await old_file_repo.get_all()
codes = [f"----- {code.filename}\n```{code.content}```" for code in old_codes]
old_codes = await self.repo.srcs.get_all()
codes = [
f"### File Name: `{code.filename}`\n```{get_markdown_code_block_type(code.filename)}\n{code.content}```\n"
for code in old_codes
]
return "\n".join(codes)

13
metagpt/actions/write_code_review.py
View file

@ -7,16 +7,17 @@
@Modified By: mashenquan, 2023/11/27. Following the think-act principle, solidify the task parameters when creating the
WriteCode object, rather than passing them in when calling the run function.
"""
from typing import Optional
from pydantic import Field
from pydantic import BaseModel, Field
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions import WriteCode
from metagpt.actions.action import Action
from metagpt.const import REQUIREMENT_FILENAME
from metagpt.logs import logger
from metagpt.schema import CodingContext
from metagpt.schema import CodingContext, Document
from metagpt.utils.common import CodeParser
from metagpt.utils.project_repo import ProjectRepo
PROMPT_TEMPLATE = """
# System
@ -126,6 +127,8 @@ or
class WriteCodeReview(Action):
name: str = "WriteCodeReview"
i_context: CodingContext = Field(default_factory=CodingContext)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
async def write_code_review_and_rewrite(self, context_prompt, cr_prompt, filename):
@ -150,7 +153,7 @@ class WriteCodeReview(Action):
code_context = await WriteCode.get_codes(
self.i_context.task_doc,
exclude=self.i_context.filename,
project_repo=self.repo.with_src_path(self.context.src_workspace),
project_repo=self.repo,
use_inc=self.config.inc,
)
@ -160,7 +163,7 @@ class WriteCodeReview(Action):
"## Code Files\n" + code_context + "\n",
]
if self.config.inc:
requirement_doc = await self.repo.docs.get(filename=REQUIREMENT_FILENAME)
requirement_doc = await Document.load(filename=self.input_args.requirements_filename)
insert_ctx_list = [
"## User New Requirements\n" + str(requirement_doc) + "\n",
"## Code Plan And Change\n" + str(self.i_context.code_plan_and_change_doc) + "\n",

47
metagpt/actions/write_prd.py
View file

@ -15,6 +15,9 @@ from __future__ import annotations
import json
from pathlib import Path
from typing import Optional
from pydantic import BaseModel, Field
from metagpt.actions import Action, ActionOutput
from metagpt.actions.action_node import ActionNode
@ -37,6 +40,7 @@ from metagpt.schema import AIMessage, Document, Documents, Message
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.mermaid import mermaid_to_file
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import DocsReporter, GalleryReporter
CONTEXT_TEMPLATE = """
@ -66,10 +70,30 @@ class WritePRD(Action):
3. Requirement update: If the requirement is an update, the PRD document will be updated.
"""
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
async def run(self, with_messages, *args, **kwargs) -> Message:
"""Run the action."""
req: Document = await self.repo.requirement
docs: list[Document] = await self.repo.docs.prd.get_all()
self.input_args = with_messages[-1].instruct_content
if not self.input_args:
self.repo = ProjectRepo(self.config.project_path)
await self.repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[-1].content)
self.input_args = AIMessage.create_instruct_value(
kvs={
"project_path": self.config.project_path,
"requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME),
"prd_filenames": [str(self.repo.docs.prd.workdir / i) for i in self.repo.docs.prd.all_files],
},
class_name="PrepareDocumentsOutput",
)
else:
self.repo = ProjectRepo(self.input_args.project_path)
req = await Document.load(filename=self.input_args.requirements_filename)
docs: list[Document] = [
await Document.load(filename=i, project_path=self.repo.workdir) for i in self.input_args.prd_filenames
]
if not req:
raise FileNotFoundError("No requirement document found.")
@ -82,10 +106,14 @@ class WritePRD(Action):
# if requirement is related to other documents, update them, otherwise create a new one
if related_docs := await self.get_related_docs(req, docs):
logger.info(f"Requirement update detected: {req.content}")
await self._handle_requirement_update(req, related_docs)
await self._handle_requirement_update(req=req, related_docs=related_docs)
else:
logger.info(f"New requirement detected: {req.content}")
await self._handle_new_requirement(req)
kvs = self.input_args.model_dump()
kvs["changed_prd_filenames"] = [
str(self.repo.docs.prd.workdir / i) for i in list(self.repo.docs.prd.changed_files.keys())
]
return AIMessage(
content="PRD is completed. "
+ "\n".join(
@ -93,6 +121,7 @@ class WritePRD(Action):
+ list(self.repo.resources.prd.changed_files.keys())
+ list(self.repo.resources.competitive_analysis.changed_files.keys())
),
instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WritePRDOutput"),
cause_by=self,
)
@ -103,6 +132,14 @@ class WritePRD(Action):
return AIMessage(
content=f"A new issue is received: {BUGFIX_FILENAME}",
cause_by=FixBug,
instruct_content=AIMessage.create_instruct_value(
{
"project_path": str(self.repo.workdir),
"issue_filename": str(self.repo.docs.workdir / BUGFIX_FILENAME),
"requirements_filename": str(self.repo.docs.workdir / REQUIREMENT_FILENAME),
},
class_name="IssueDetail",
),
send_to="Alex", # the name of Engineer
)
@ -128,7 +165,7 @@ class WritePRD(Action):
async def _handle_requirement_update(self, req: Document, related_docs: list[Document]) -> ActionOutput:
# ... requirement update logic ...
for doc in related_docs:
await self._update_prd(req, doc)
await self._update_prd(req=req, prd_doc=doc)
return Documents.from_iterable(documents=related_docs).to_action_output()
async def _is_bugfix(self, context: str) -> bool:
@ -159,7 +196,7 @@ class WritePRD(Action):
async def _update_prd(self, req: Document, prd_doc: Document) -> Document:
async with DocsReporter(enable_llm_stream=True) as reporter:
await reporter.async_report({"type": "prd"}, "meta")
new_prd_doc: Document = await self._merge(req, prd_doc)
new_prd_doc: Document = await self._merge(req=req, related_doc=prd_doc)
await self.repo.docs.prd.save_doc(doc=new_prd_doc)
await self._save_competitive_analysis(new_prd_doc)
md = await self.repo.resources.prd.save_pdf(doc=new_prd_doc)

4
metagpt/actions/write_prd_an.py
View file

@ -5,7 +5,7 @@
@Author : alexanderwu
@File : write_prd_an.py
"""
from typing import List
from typing import List, Union
from metagpt.actions.action_node import ActionNode
@ -132,7 +132,7 @@ REQUIREMENT_ANALYSIS = ActionNode(
REFINED_REQUIREMENT_ANALYSIS = ActionNode(
key="Refined Requirement Analysis",
expected_type=List[str],
expected_type=Union[List[str], str],
instruction="Review and refine the existing requirement analysis into a string list to align with the evolving needs of the project "
"due to incremental development. Ensure the analysis comprehensively covers the new features and enhancements "
"required for the refined project scope.",

213
metagpt/roles/engineer.py
View file

@ -24,20 +24,15 @@ from collections import defaultdict
from pathlib import Path
from typing import List, Optional, Set
from metagpt.actions import (
Action,
UserRequirement,
WriteCode,
WriteCodeReview,
WriteTasks,
)
from pydantic import BaseModel, Field
from metagpt.actions import UserRequirement, WriteCode, WriteCodeReview, WriteTasks
from metagpt.actions.fix_bug import FixBug
from metagpt.actions.prepare_documents import PrepareDocuments
from metagpt.actions.project_management_an import REFINED_TASK_LIST, TASK_LIST
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.actions.write_code_plan_and_change_an import WriteCodePlanAndChange
from metagpt.const import (
BUGFIX_FILENAME,
CODE_PLAN_AND_CHANGE_FILE_REPO,
MESSAGE_ROUTE_TO_SELF,
REQUIREMENT_FILENAME,
@ -63,6 +58,7 @@ from metagpt.utils.common import (
init_python_folder,
)
from metagpt.utils.git_repository import ChangeType
from metagpt.utils.project_repo import ProjectRepo
IS_PASS_PROMPT = """
{context}
@ -100,6 +96,8 @@ class Engineer(Role):
summarize_todos: list = []
next_todo_action: str = ""
n_summarize: int = 0
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
def __init__(self, **kwargs) -> None:
super().__init__(**kwargs)
@ -139,14 +137,20 @@ class Engineer(Role):
coding_context = await todo.run()
# Code review
if review:
action = WriteCodeReview(i_context=coding_context, context=self.context, llm=self.llm)
action = WriteCodeReview(
i_context=coding_context,
repo=self.repo,
input_args=self.input_args,
context=self.context,
llm=self.llm,
)
self._init_action(action)
coding_context = await action.run()
dependencies = {coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}
if self.config.inc:
dependencies.add(coding_context.code_plan_and_change_doc.root_relative_path)
await self.project_repo.srcs.save(
await self.repo.srcs.save(
filename=coding_context.filename,
dependencies=list(dependencies),
content=coding_context.code_doc.content,
@ -186,9 +190,9 @@ class Engineer(Role):
summary_filename = Path(todo.i_context.design_filename).with_suffix(".md").name
dependencies = {todo.i_context.design_filename, todo.i_context.task_filename}
for filename in todo.i_context.codes_filenames:
rpath = self.project_repo.src_relative_path / filename
rpath = self.repo.src_relative_path / filename
dependencies.add(str(rpath))
await self.project_repo.resources.code_summary.save(
await self.repo.resources.code_summary.save(
filename=summary_filename, content=summary, dependencies=dependencies
)
is_pass, reason = await self._is_pass(summary)
@ -196,23 +200,39 @@ class Engineer(Role):
todo.i_context.reason = reason
tasks.append(todo.i_context.model_dump())
await self.project_repo.docs.code_summary.save(
await self.repo.docs.code_summary.save(
filename=Path(todo.i_context.design_filename).name,
content=todo.i_context.model_dump_json(),
dependencies=dependencies,
)
else:
await self.project_repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name)
await self.repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name)
self.summarize_todos = []
logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}")
if not tasks or self.config.max_auto_summarize_code == 0:
self.n_summarize = 0
kvs = self.input_args.model_dump()
kvs["changed_src_filenames"] = [
str(self.repo.srcs.workdir / i) for i in list(self.repo.srcs.changed_files.keys())
]
if self.repo.docs.code_plan_and_change.changed_files:
kvs["changed_code_plan_and_change_filenames"] = [
str(self.repo.docs.code_plan_and_change.workdir / i)
for i in list(self.repo.docs.code_plan_and_change.changed_files.keys())
]
if self.repo.docs.code_summary.changed_files:
kvs["changed_code_summary_filenames"] = [
str(self.repo.docs.code_summary.workdir / i)
for i in list(self.repo.docs.code_summary.changed_files.keys())
]
return AIMessage(
content=f"Coding is complete. The source code is at {self.project_repo.workdir.name}/{self.project_repo.srcs.root_path}, containing: "
content=f"Coding is complete. The source code is at {self.repo.workdir.name}/{self.repo.srcs.root_path}, containing: "
+ "\n".join(
list(self.project_repo.resources.code_summary.changed_files.keys())
+ list(self.project_repo.srcs.changed_files.keys())
list(self.repo.resources.code_summary.changed_files.keys())
+ list(self.repo.srcs.changed_files.keys())
+ list(self.repo.resources.code_plan_and_change.changed_files.keys())
),
instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="SummarizeCodeOutput"),
cause_by=SummarizeCode,
send_to="Edward", # The name of QaEngineer
)
@ -227,15 +247,15 @@ class Engineer(Role):
code_plan_and_change = node.instruct_content.model_dump_json()
dependencies = {
REQUIREMENT_FILENAME,
str(self.project_repo.docs.prd.root_path / self.rc.todo.i_context.prd_filename),
str(self.project_repo.docs.system_design.root_path / self.rc.todo.i_context.design_filename),
str(self.project_repo.docs.task.root_path / self.rc.todo.i_context.task_filename),
str(Path(self.rc.todo.i_context.prd_filename).relative_to(self.repo.workdir)),
str(Path(self.rc.todo.i_context.design_filename).relative_to(self.repo.workdir)),
str(Path(self.rc.todo.i_context.task_filename).relative_to(self.repo.workdir)),
}
code_plan_and_change_filepath = Path(self.rc.todo.i_context.design_filename)
await self.project_repo.docs.code_plan_and_change.save(
await self.repo.docs.code_plan_and_change.save(
filename=code_plan_and_change_filepath.name, content=code_plan_and_change, dependencies=dependencies
)
await self.project_repo.resources.code_plan_and_change.save(
await self.repo.resources.code_plan_and_change.save(
filename=code_plan_and_change_filepath.with_suffix(".md").name,
content=node.content,
dependencies=dependencies,
@ -250,10 +270,11 @@ class Engineer(Role):
return True, rsp
return False, rsp
async def _think(self) -> Action | None:
async def _think(self) -> bool:
if not self.rc.news:
return None
return False
msg = self.rc.news[0]
input_args = msg.instruct_content
if msg.cause_by == any_to_str(UserRequirement):
self.rc.todo = PrepareDocuments(
key_descriptions={
@ -263,42 +284,47 @@ class Engineer(Role):
context=self.context,
send_to=any_to_str(self),
)
return self.rc.todo
if not self.src_workspace:
self.src_workspace = get_project_srcs_path(self.project_repo.workdir)
self.repo = ProjectRepo(input_args.project_path)
self.input_args = input_args
return bool(self.rc.todo)
elif msg.cause_by in {any_to_str(WriteTasks), any_to_str(FixBug)}:
self.input_args = input_args
self.repo = ProjectRepo(input_args.project_path)
if self.repo.src_relative_path is None:
path = get_project_srcs_path(self.repo.workdir)
self.repo.with_src_path(path)
write_plan_and_change_filters = any_to_str_set([PrepareDocuments, WriteTasks, FixBug])
write_code_filters = any_to_str_set([WriteTasks, WriteCodePlanAndChange, SummarizeCode])
summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview])
if self.config.inc and msg.cause_by in write_plan_and_change_filters:
logger.debug(f"TODO WriteCodePlanAndChange:{msg.model_dump_json()}")
await self._new_code_plan_and_change_action(cause_by=msg.cause_by)
return self.rc.todo
return bool(self.rc.todo)
if msg.cause_by in write_code_filters:
logger.debug(f"TODO WriteCode:{msg.model_dump_json()}")
await self._new_code_actions()
return self.rc.todo
return bool(self.rc.todo)
if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self):
logger.debug(f"TODO SummarizeCode:{msg.model_dump_json()}")
await self._new_summarize_actions()
return self.rc.todo
return None
return bool(self.rc.todo)
return False
async def _new_coding_context(self, filename, dependency) -> Optional[CodingContext]:
old_code_doc = await self.project_repo.srcs.get(filename)
old_code_doc = await self.repo.srcs.get(filename)
if not old_code_doc:
old_code_doc = Document(root_path=str(self.project_repo.src_relative_path), filename=filename, content="")
old_code_doc = Document(root_path=str(self.repo.src_relative_path), filename=filename, content="")
dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)}
task_doc = None
design_doc = None
code_plan_and_change_doc = await self._get_any_code_plan_and_change() if await self._is_fixbug() else None
for i in dependencies:
if str(i.parent) == TASK_FILE_REPO:
task_doc = await self.project_repo.docs.task.get(i.name)
task_doc = await self.repo.docs.task.get(i.name)
elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO:
design_doc = await self.project_repo.docs.system_design.get(i.name)
design_doc = await self.repo.docs.system_design.get(i.name)
elif str(i.parent) == CODE_PLAN_AND_CHANGE_FILE_REPO:
code_plan_and_change_doc = await self.project_repo.docs.code_plan_and_change.get(i.name)
code_plan_and_change_doc = await self.repo.docs.code_plan_and_change.get(i.name)
if not task_doc or not design_doc:
if filename == "__init__.py": # `__init__.py` created by `init_python_folder`
return None
@ -318,34 +344,66 @@ class Engineer(Role):
if not context:
return None # `__init__.py` created by `init_python_folder`
coding_doc = Document(
root_path=str(self.project_repo.src_relative_path), filename=filename, content=context.model_dump_json()
root_path=str(self.repo.src_relative_path), filename=filename, content=context.model_dump_json()
)
return coding_doc
async def _new_code_actions(self):
bug_fix = await self._is_fixbug()
# Prepare file repos
changed_src_files = self.project_repo.srcs.changed_files
changed_src_files = self.repo.srcs.changed_files
if self.context.kwargs.src_filename:
changed_src_files = {self.context.kwargs.src_filename: ChangeType.UNTRACTED}
if bug_fix:
changed_src_files = self.project_repo.srcs.all_files
changed_task_files = self.project_repo.docs.task.changed_files
changed_src_files = self.repo.srcs.all_files
changed_files = Documents()
# Recode caused by upstream changes.
for filename in changed_task_files:
design_doc = await self.project_repo.docs.system_design.get(filename)
task_doc = await self.project_repo.docs.task.get(filename)
code_plan_and_change_doc = await self.project_repo.docs.code_plan_and_change.get(filename)
if hasattr(self.input_args, "changed_task_filenames"):
changed_task_filenames = self.input_args.changed_task_filenames
else:
changed_task_filenames = [
str(self.repo.docs.task.workdir / i) for i in list(self.repo.docs.task.changed_files.keys())
]
for filename in changed_task_filenames:
task_filename = Path(filename)
design_filename = None
if hasattr(self.input_args, "changed_system_design_filenames"):
changed_system_design_filenames = self.input_args.changed_system_design_filenames
else:
changed_system_design_filenames = [
str(self.repo.docs.system_design.workdir / i)
for i in list(self.repo.docs.system_design.changed_files.keys())
]
for i in changed_system_design_filenames:
if task_filename.name == Path(i).name:
design_filename = Path(i)
break
code_plan_and_change_filename = None
if hasattr(self.input_args, "changed_code_plan_and_change_filenames"):
changed_code_plan_and_change_filenames = self.input_args.changed_code_plan_and_change_filenames
else:
changed_code_plan_and_change_filenames = [
str(self.repo.docs.code_plan_and_change.workdir / i)
for i in list(self.repo.docs.code_plan_and_change.changed_files.keys())
]
for i in changed_code_plan_and_change_filenames:
if task_filename.name == Path(i).name:
code_plan_and_change_filename = Path(i)
break
design_doc = await Document.load(filename=design_filename, project_path=self.repo.workdir)
task_doc = await Document.load(filename=task_filename, project_path=self.repo.workdir)
code_plan_and_change_doc = await Document.load(
filename=code_plan_and_change_filename, project_path=self.repo.workdir
)
task_list = self._parse_tasks(task_doc)
await self._init_python_folder(task_list)
for task_filename in task_list:
if self.context.kwargs.src_filename and task_filename != self.context.kwargs.src_filename:
continue
old_code_doc = await self.project_repo.srcs.get(task_filename)
old_code_doc = await self.repo.srcs.get(task_filename)
if not old_code_doc:
old_code_doc = Document(
root_path=str(self.project_repo.src_relative_path), filename=task_filename, content=""
root_path=str(self.repo.src_relative_path), filename=task_filename, content=""
)
if not code_plan_and_change_doc:
context = CodingContext(
@ -360,7 +418,7 @@ class Engineer(Role):
code_plan_and_change_doc=code_plan_and_change_doc,
)
coding_doc = Document(
root_path=str(self.project_repo.src_relative_path),
root_path=str(self.repo.src_relative_path),
filename=task_filename,
content=context.model_dump_json(),
)
@ -371,10 +429,11 @@ class Engineer(Role):
)
changed_files.docs[task_filename] = coding_doc
self.code_todos = [
WriteCode(i_context=i, context=self.context, llm=self.llm) for i in changed_files.docs.values()
WriteCode(i_context=i, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm)
for i in changed_files.docs.values()
]
# Code directly modified by the user.
dependency = await self.git_repo.get_dependency()
dependency = await self.repo.git_repo.get_dependency()
for filename in changed_src_files:
if filename in changed_files.docs:
continue
@ -382,24 +441,30 @@ class Engineer(Role):
if not coding_doc:
continue # `__init__.py` created by `init_python_folder`
changed_files.docs[filename] = coding_doc
self.code_todos.append(WriteCode(i_context=coding_doc, context=self.context, llm=self.llm))
self.code_todos.append(
WriteCode(
i_context=coding_doc, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm
)
)
if self.code_todos:
self.set_todo(self.code_todos[0])
async def _new_summarize_actions(self):
src_files = self.project_repo.srcs.all_files
src_files = self.repo.srcs.all_files
# Generate a SummarizeCode action for each pair of (system_design_doc, task_doc).
summarizations = defaultdict(list)
for filename in src_files:
dependencies = await self.project_repo.srcs.get_dependency(filename=filename)
dependencies = await self.repo.srcs.get_dependency(filename=filename)
ctx = CodeSummarizeContext.loads(filenames=list(dependencies))
summarizations[ctx].append(filename)
for ctx, filenames in summarizations.items():
if not ctx.design_filename or not ctx.task_filename:
continue # cause by `__init__.py` which is created by `init_python_folder`
ctx.codes_filenames = filenames
new_summarize = SummarizeCode(i_context=ctx, context=self.context, llm=self.llm)
new_summarize = SummarizeCode(
i_context=ctx, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm
)
for i, act in enumerate(self.summarize_todos):
if act.i_context.task_filename == new_summarize.i_context.task_filename:
self.summarize_todos[i] = new_summarize
@ -412,16 +477,37 @@ class Engineer(Role):
async def _new_code_plan_and_change_action(self, cause_by: str):
"""Create a WriteCodePlanAndChange action for subsequent to-do actions."""
files = self.project_repo.all_files
options = {}
if cause_by != any_to_str(FixBug):
requirement_doc = await self.project_repo.docs.get(REQUIREMENT_FILENAME)
requirement_doc = await Document.load(filename=self.input_args.requirements_filename)
options["requirement"] = requirement_doc.content
else:
fixbug_doc = await self.project_repo.docs.get(BUGFIX_FILENAME)
fixbug_doc = await Document.load(filename=self.input_args.issue_filename)
options["issue"] = fixbug_doc.content
code_plan_and_change_ctx = CodePlanAndChangeContext.loads(files, **options)
self.rc.todo = WriteCodePlanAndChange(i_context=code_plan_and_change_ctx, context=self.context, llm=self.llm)
# The code here is flawed: if there are multiple unrelated requirements, this piece of logic will break
if hasattr(self.input_args, "changed_prd_filenames"):
code_plan_and_change_ctx = CodePlanAndChangeContext(
requirement=options.get("requirement", ""),
issue=options.get("issue", ""),
prd_filename=self.input_args.changed_prd_filenames[0],
design_filename=self.input_args.changed_system_design_filenames[0],
task_filename=self.input_args.changed_task_filenames[0],
)
else:
code_plan_and_change_ctx = CodePlanAndChangeContext(
requirement=options.get("requirement", ""),
issue=options.get("issue", ""),
prd_filename=str(self.repo.docs.prd.workdir / self.repo.docs.prd.all_files[0]),
design_filename=str(self.repo.docs.system_design.workdir / self.repo.docs.system_design.all_files[0]),
task_filename=str(self.repo.docs.task.workdir / self.repo.docs.task.all_files[0]),
)
self.rc.todo = WriteCodePlanAndChange(
i_context=code_plan_and_change_ctx,
repo=self.repo,
input_args=self.input_args,
context=self.context,
llm=self.llm,
)
@property
def action_description(self) -> str:
@ -433,17 +519,16 @@ class Engineer(Role):
filename = Path(i)
if filename.suffix != ".py":
continue
workdir = self.src_workspace / filename.parent
workdir = self.repo.srcs.workdir / filename.parent
await init_python_folder(workdir)
async def _is_fixbug(self) -> bool:
fixbug_doc = await self.project_repo.docs.get(BUGFIX_FILENAME)
return bool(fixbug_doc and fixbug_doc.content)
return bool(self.input_args and hasattr(self.input_args, "issue_filename"))
async def _get_any_code_plan_and_change(self) -> Optional[Document]:
changed_files = self.project_repo.docs.code_plan_and_change.changed_files
changed_files = self.repo.docs.code_plan_and_change.changed_files
for filename in changed_files.keys():
doc = await self.project_repo.docs.code_plan_and_change.get(filename)
doc = await self.repo.docs.code_plan_and_change.get(filename)
if doc and doc.content:
return doc
return None

7
metagpt/roles/product_manager.py
View file

@ -7,10 +7,12 @@
@Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135.
"""
from metagpt.actions import UserRequirement, WritePRD
from metagpt.actions.prepare_documents import PrepareDocuments
from metagpt.roles.role import Role, RoleReactMode
from metagpt.roles.role import Role
from metagpt.utils.common import any_to_name, any_to_str
from metagpt.utils.git_repository import GitRepository
class ProductManager(Role):
@ -35,12 +37,11 @@ class ProductManager(Role):
self.enable_memory = False
self.set_actions([PrepareDocuments(send_to=any_to_str(self)), WritePRD])
self._watch([UserRequirement, PrepareDocuments])
self.rc.react_mode = RoleReactMode.BY_ORDER
self.todo_action = any_to_name(WritePRD)
async def _think(self) -> bool:
"""Decide what to do"""
if self.git_repo and not self.config.git_reinit:
if GitRepository.is_git_dir(self.config.project_path) and not self.config.git_reinit:
self._set_state(1)
else:
self._set_state(0)

95
metagpt/roles/qa_engineer.py
View file

@ -14,6 +14,9 @@
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
from typing import Optional
from pydantic import BaseModel, Field
from metagpt.actions import DebugError, RunCode, UserRequirement, WriteTest
from metagpt.actions.prepare_documents import PrepareDocuments
@ -25,9 +28,11 @@ from metagpt.schema import AIMessage, Document, Message, RunCodeContext, Testing
from metagpt.utils.common import (
any_to_str,
any_to_str_set,
get_project_srcs_path,
init_python_folder,
parse_recipient,
)
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.report import EditorReporter
@ -41,6 +46,8 @@ class QaEngineer(Role):
)
test_round_allowed: int = 5
test_round: int = 0
repo: Optional[ProjectRepo] = Field(default=None, exclude=True)
input_args: Optional[BaseModel] = Field(default=None, exclude=True)
def __init__(self, **kwargs):
super().__init__(**kwargs)
@ -57,22 +64,21 @@ class QaEngineer(Role):
self.test_round = 0
async def _write_test(self, message: Message) -> None:
src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs
reqa_file = self.context.kwargs.reqa_file or self.config.reqa_file
changed_files = {reqa_file} if reqa_file else set(src_file_repo.changed_files.keys())
changed_files = {reqa_file} if reqa_file else set(self.repo.srcs.changed_files.keys())
for filename in changed_files:
# write tests
if not filename or "test" in filename:
continue
code_doc = await src_file_repo.get(filename)
if not code_doc:
code_doc = await self.repo.srcs.get(filename)
if not code_doc or not code_doc.content:
continue
if not code_doc.filename.endswith(".py"):
continue
test_doc = await self.project_repo.tests.get("test_" + code_doc.filename)
test_doc = await self.repo.tests.get("test_" + code_doc.filename)
if not test_doc:
test_doc = Document(
root_path=str(self.project_repo.tests.root_path), filename="test_" + code_doc.filename, content=""
root_path=str(self.repo.tests.root_path), filename="test_" + code_doc.filename, content=""
)
logger.info(f"Writing {test_doc.filename}..")
context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc)
@ -81,40 +87,38 @@ class QaEngineer(Role):
async with EditorReporter(enable_llm_stream=True) as reporter:
await reporter.async_report({"type": "test", "filename": test_doc.filename}, "meta")
doc = await self.project_repo.tests.save_doc(
doc = await self.repo.tests.save_doc(
doc=context.test_doc, dependencies={context.code_doc.root_relative_path}
)
await reporter.async_report(self.project_repo.workdir / doc.root_relative_path, "path")
await reporter.async_report(self.repo.workdir / doc.root_relative_path, "path")
# prepare context for run tests in next round
run_code_context = RunCodeContext(
command=["python", context.test_doc.root_relative_path],
code_filename=context.code_doc.filename,
test_filename=context.test_doc.filename,
working_directory=str(self.project_repo.workdir),
working_directory=str(self.repo.workdir),
additional_python_paths=[str(self.context.src_workspace)],
)
self.publish_message(
AIMessage(content=run_code_context.model_dump_json(), cause_by=WriteTest, send_to=MESSAGE_ROUTE_TO_SELF)
)
logger.info(f"Done {str(self.project_repo.tests.workdir)} generating.")
logger.info(f"Done {str(self.repo.tests.workdir)} generating.")
async def _run_code(self, msg):
run_code_context = RunCodeContext.loads(msg.content)
src_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get(
run_code_context.code_filename
)
src_doc = await self.repo.srcs.get(run_code_context.code_filename)
if not src_doc:
return
test_doc = await self.project_repo.tests.get(run_code_context.test_filename)
test_doc = await self.repo.tests.get(run_code_context.test_filename)
if not test_doc:
return
run_code_context.code = src_doc.content
run_code_context.test_code = test_doc.content
result = await RunCode(i_context=run_code_context, context=self.context, llm=self.llm).run()
run_code_context.output_filename = run_code_context.test_filename + ".json"
await self.project_repo.test_outputs.save(
await self.repo.test_outputs.save(
filename=run_code_context.output_filename,
content=result.model_dump_json(),
dependencies={src_doc.root_relative_path, test_doc.root_relative_path},
@ -124,31 +128,53 @@ class QaEngineer(Role):
# the recipient might be Engineer or myself
recipient = parse_recipient(result.summary)
mappings = {"Engineer": "Alex", "QaEngineer": "Edward"}
self.publish_message(
AIMessage(
content=run_code_context.model_dump_json(),
cause_by=RunCode,
send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE),
if recipient != "Engineer":
self.publish_message(
AIMessage(
content=run_code_context.model_dump_json(),
cause_by=RunCode,
instruct_content=self.input_args,
send_to=MESSAGE_ROUTE_TO_SELF,
)
)
else:
kvs = self.input_args.model_dump()
kvs["changed_test_filenames"] = [
str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys())
]
self.publish_message(
AIMessage(
content=run_code_context.model_dump_json(),
cause_by=RunCode,
instruct_content=self.input_args,
send_to=mappings.get(recipient, MESSAGE_ROUTE_TO_NONE),
)
)
)
async def _debug_error(self, msg):
run_code_context = RunCodeContext.loads(msg.content)
code = await DebugError(i_context=run_code_context, context=self.context, llm=self.llm).run()
await self.project_repo.tests.save(filename=run_code_context.test_filename, content=code)
code = await DebugError(
i_context=run_code_context, repo=self.repo, input_args=self.input_args, context=self.context, llm=self.llm
).run()
await self.repo.tests.save(filename=run_code_context.test_filename, content=code)
run_code_context.output = None
self.publish_message(
AIMessage(content=run_code_context.model_dump_json(), cause_by=DebugError, send_to=MESSAGE_ROUTE_TO_SELF)
)
async def _act(self) -> Message:
if self.project_path:
await init_python_folder(self.project_repo.tests.workdir)
if self.input_args.project_path:
await init_python_folder(self.repo.tests.workdir)
if self.test_round > self.test_round_allowed:
kvs = self.input_args.model_dump()
kvs["changed_test_filenames"] = [
str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys())
]
result_msg = AIMessage(
content=f"Exceeding {self.test_round_allowed} rounds of tests, stop. "
+ "\n".join(list(self.project_repo.tests.changed_files.keys())),
+ "\n".join(list(self.repo.tests.changed_files.keys())),
cause_by=WriteTest,
instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTestOutput"),
send_to=MESSAGE_ROUTE_TO_NONE,
)
return result_msg
@ -171,8 +197,13 @@ class QaEngineer(Role):
elif msg.cause_by == any_to_str(UserRequirement):
return await self._parse_user_requirement(msg)
self.test_round += 1
kvs = self.input_args.model_dump()
kvs["changed_test_filenames"] = [
str(self.repo.tests.workdir / i) for i in list(self.repo.tests.changed_files.keys())
]
return AIMessage(
content=f"Round {self.test_round} of tests done",
instruct_content=AIMessage.create_instruct_value(kvs=kvs, class_name="WriteTestOutput"),
cause_by=WriteTest,
send_to=MESSAGE_ROUTE_TO_NONE,
)
@ -190,3 +221,15 @@ class QaEngineer(Role):
if not self.src_workspace:
self.src_workspace = self.git_repo.workdir / self.git_repo.workdir.name
return rsp
async def _think(self) -> bool:
if not self.rc.news:
return False
msg = self.rc.news[0]
if msg.cause_by == any_to_str(SummarizeCode):
self.input_args = msg.instruct_content
self.repo = ProjectRepo(self.input_args.project_path)
if self.repo.src_relative_path is None:
path = get_project_srcs_path(self.repo.workdir)
self.repo.with_src_path(path)
return True

24
metagpt/roles/role.py
View file

@ -45,7 +45,6 @@ from metagpt.schema import (
)
from metagpt.strategy.planner import Planner
from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator
from metagpt.utils.project_repo import ProjectRepo
from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output
if TYPE_CHECKING:
@ -196,29 +195,6 @@ class Role(SerializationMixin, ContextMixin, BaseModel):
value.context = self.context
self.rc.todo = value
@property
def git_repo(self):
"""Git repo"""
return self.context.git_repo
@git_repo.setter
def git_repo(self, value):
self.context.git_repo = value
@property
def src_workspace(self):
"""Source workspace under git repo"""
return self.context.src_workspace
@src_workspace.setter
def src_workspace(self, value):
self.context.src_workspace = value
@property
def project_repo(self) -> ProjectRepo:
project_repo = ProjectRepo(self.context.git_repo)
return project_repo.with_src_path(self.context.src_workspace) if self.context.src_workspace else project_repo
@property
def prompt_schema(self):
"""Prompt schema: json/markdown"""

66
metagpt/schema.py
View file

@ -31,6 +31,7 @@ from pydantic import (
ConfigDict,
Field,
PrivateAttr,
create_model,
field_serializer,
field_validator,
model_serializer,
@ -43,13 +44,18 @@ from metagpt.const import (
MESSAGE_ROUTE_FROM,
MESSAGE_ROUTE_TO,
MESSAGE_ROUTE_TO_ALL,
PRDS_FILE_REPO,
SYSTEM_DESIGN_FILE_REPO,
TASK_FILE_REPO,
)
from metagpt.logs import logger
from metagpt.repo_parser import DotClassInfo
from metagpt.utils.common import CodeParser, any_to_str, any_to_str_set, import_class
from metagpt.utils.common import (
CodeParser,
any_to_str,
any_to_str_set,
aread,
import_class,
)
from metagpt.utils.exceptions import handle_exception
from metagpt.utils.report import TaskReporter
from metagpt.utils.serialize import (
@ -157,6 +163,30 @@ class Document(BaseModel):
def __repr__(self):
return self.content
@classmethod
async def load(
cls, filename: Union[str, Path], project_path: Optional[Union[str, Path]] = None
) -> Optional["Document"]:
"""
Load a document from a file.
Args:
filename (Union[str, Path]): The path to the file to load.
project_path (Optional[Union[str, Path]], optional): The path to the project. Defaults to None.
Returns:
Optional[Document]: The loaded document, or None if the file does not exist.
"""
if not filename or not Path(filename).exists():
return None
content = await aread(filename=filename)
doc = cls(content=content, filename=str(filename))
if project_path and Path(filename).is_relative_to(project_path):
doc.root_path = Path(filename).relative_to(project_path).parent
doc.filename = Path(filename).name
return doc
class Documents(BaseModel):
"""A class representing a collection of documents.
@ -360,6 +390,22 @@ class Message(BaseModel):
def add_metadata(self, key: str, value: str):
self.metadata[key] = value
@staticmethod
def create_instruct_value(kvs: Dict[str, Any], class_name: str = "") -> BaseModel:
"""
Dynamically creates a Pydantic BaseModel subclass based on a given dictionary.
Parameters:
- data: A dictionary from which to create the BaseModel subclass.
Returns:
- A Pydantic BaseModel subclass instance populated with the given data.
"""
if not class_name:
class_name = "DM" + uuid.uuid4().hex[0:8]
dynamic_class = create_model(class_name, **{key: (value.__class__, ...) for key, value in kvs.items()})
return dynamic_class.model_validate(kvs)
class UserMessage(Message):
"""便于支持OpenAI的消息
@ -762,22 +808,6 @@ class CodePlanAndChangeContext(BaseModel):
design_filename: str = ""
task_filename: str = ""
@staticmethod
def loads(filenames: List, **kwargs) -> CodePlanAndChangeContext:
ctx = CodePlanAndChangeContext(requirement=kwargs.get("requirement", ""), issue=kwargs.get("issue", ""))
for filename in filenames:
filename = Path(filename)
if filename.is_relative_to(PRDS_FILE_REPO):
ctx.prd_filename = filename.name
continue
if filename.is_relative_to(SYSTEM_DESIGN_FILE_REPO):
ctx.design_filename = filename.name
continue
if filename.is_relative_to(TASK_FILE_REPO):
ctx.task_filename = filename.name
continue
return ctx
# mermaid class view
class UMLClassMeta(BaseModel):

41
metagpt/utils/common.py
View file

@ -899,3 +899,44 @@ async def init_python_folder(workdir: str | Path):
return
async with aiofiles.open(init_filename, "a"):
os.utime(init_filename, None)
def get_markdown_code_block_type(filename: str) -> str:
if not filename:
return ""
ext = Path(filename).suffix
types = {
".py": "python",
".js": "javascript",
".java": "java",
".cpp": "cpp",
".c": "c",
".html": "html",
".css": "css",
".xml": "xml",
".json": "json",
".yaml": "yaml",
".md": "markdown",
".sql": "sql",
".rb": "ruby",
".php": "php",
".sh": "bash",
".swift": "swift",
".go": "go",
".rs": "rust",
".pl": "perl",
".asm": "assembly",
".r": "r",
".scss": "scss",
".sass": "sass",
".lua": "lua",
".ts": "typescript",
".tsx": "tsx",
".jsx": "jsx",
".yml": "yaml",
".ini": "ini",
".toml": "toml",
".svg": "xml", # SVG can often be treated as XML
# Add more file extensions and corresponding code block types as needed
}
return types.get(ext, "")

2
metagpt/utils/git_repository.py
View file

@ -156,6 +156,8 @@ class GitRepository:
:param local_path: The local path to check.
:return: True if the directory is a Git repository, False otherwise.
"""
if not local_path:
return False
git_dir = Path(local_path) / ".git"
if git_dir.exists() and is_git_dir(git_dir):
return True

9
metagpt/utils/project_repo.py
View file

@ -140,10 +140,11 @@ class ProjectRepo(FileRepository):
return bool(code_files)
def with_src_path(self, path: str | Path) -> ProjectRepo:
try:
self._srcs_path = Path(path).relative_to(self.workdir)
except ValueError:
self._srcs_path = Path(path)
path = Path(path)
if path.is_relative_to(self.workdir):
self._srcs_path = path.relative_to(self.workdir)
else:
self._srcs_path = path
return self
@property

3
tests/metagpt/actions/test_action_node.py
View file

@ -303,5 +303,4 @@ def test_action_node_from_pydantic_and_print_everything():
if __name__ == "__main__":
test_create_model_class()
test_create_model_class_with_mapping()
pytest.main([__file__, "-s"])

6
tests/metagpt/test_schema.py
View file

@ -392,5 +392,11 @@ async def test_parse_resources(context, content: str, key_descriptions):
assert k in result
@pytest.mark.parametrize(("name", "value"), [("c1", {"age": 10, "name": "Alice"}), ("", {"path": __file__})])
def test_create_instruct_value(name, value):
obj = Message.create_instruct_value(kvs=value, class_name=name)
assert obj.model_dump() == value
if __name__ == "__main__":
pytest.main([__file__, "-s"])