apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-04-26 17:26:22 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: +SummarizeCode, refactor project_name

Browse source
This commit is contained in:
莘权 马 2023-12-04 23:04:07 +08:00
parent 838b3cfcc8
commit 9d84c8f047
31 changed files with 671 additions and 245 deletions
Download patch file Download diff file Expand all files Collapse all files

65
metagpt/actions/design_api.py
View file

@ -7,6 +7,7 @@
@Modified By: mashenquan, 2023/11/27.
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
"""
import json
from pathlib import Path
@ -23,7 +24,6 @@ from metagpt.const import (
)
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.get_template import get_template
from metagpt.utils.mermaid import mermaid_to_file
@ -43,7 +43,7 @@ Requirement: Fill in the following missing information based on the context, eac
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks.
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## Project name: Constant text.
## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
@ -58,15 +58,15 @@ and only output the json inside this tag, nothing else
""",
"FORMAT_EXAMPLE": """
[CONTENT]
{
{{
"Implementation approach": "We will ...",
"project_name": "snake_game",
"Project name": "{project_name}",
"File list": ["main.py"],
"Data structures and interfaces": '
classDiagram
class Game{
class Game{{
+int score
}
}}
...
Game "1" -- "1" Food: has
',
@ -77,7 +77,7 @@ and only output the json inside this tag, nothing else
G->>M: end game
',
"Anything UNCLEAR": "The requirement is clear to me."
}
}}
[/CONTENT]
""",
},
@ -96,7 +96,7 @@ ATTENTION: Output carefully referenced "Format example" in format.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## Project name: Constant text.
## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
@ -112,9 +112,9 @@ ATTENTION: Output carefully referenced "Format example" in format.
## Implementation approach
We will ...
## project_name
## Project name
```python
"snake_game"
"{project_name}"
```
## File list
@ -151,7 +151,7 @@ The requirement is clear to me.
OUTPUT_MAPPING = {
"Implementation approach": (str, ...),
"project_name": (str, ...),
"Project name": (str, ...),
"File list": (List[str], ...),
"Data structures and interfaces": (str, ...),
"Program call flow": (str, ...),
@ -173,7 +173,7 @@ ATTENTION: Output carefully referenced "Old Design" in format.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## Project name: Constant text "{project_name}".
## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
@ -229,50 +229,21 @@ class WriteDesign(Action):
async def _new_system_design(self, context, format=CONFIG.prompt_format):
prompt_template, format_example = get_template(templates, format)
format_example = format_example.format(project_name=CONFIG.project_name)
prompt = prompt_template.format(context=context, format_example=format_example)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
self._rename_project_name(system_design=system_design)
await self._rename_workspace(system_design)
return system_design
async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):
prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content)
prompt = MERGE_PROMPT.format(
old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name
)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
# fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python
# package name" contain space, have to use setattr
self._rename_project_name(system_design=system_design)
system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False)
return system_design_doc
@staticmethod
def _rename_project_name(system_design):
# fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name"
# contain space, have to use setattr
if CONFIG.project_name:
setattr(
system_design.instruct_content,
"project_name",
CONFIG.project_name,
)
return
setattr(
system_design.instruct_content,
"project_name",
system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'),
)
@staticmethod
async def _rename_workspace(system_design):
if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to
# Section 2.2.3.10 of RFC 135
return
if isinstance(system_design, ActionOutput):
ws_name = system_design.instruct_content.dict()["project_name"]
else:
ws_name = CodeParser.parse_str(block="project_name", text=system_design)
CONFIG.git_repo.rename_root(ws_name)
async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document:
prd = await prds_file_repo.get(filename)
old_system_design_doc = await system_design_file_repo.get(filename)
@ -296,10 +267,10 @@ class WriteDesign(Action):
@staticmethod
async def _save_data_api_design(design_doc):
m = json.loads(design_doc.content)
data_api_design = m.get("Data structures and interface definitions")
data_api_design = m.get("Data structures and interfaces")
if not data_api_design:
return
pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("")
pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
await WriteDesign._save_mermaid_file(data_api_design, pathname)
logger.info(f"Save class view to {str(pathname)}")

7
metagpt/actions/prepare_documents.py
View file

@ -3,7 +3,7 @@
"""
@Time : 2023/11/20
@Author : mashenquan
@File : git_repository.py
@File : prepare_documents.py
@Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt.
RFC 135 2.2.3.5.1.
"""
@ -26,7 +26,10 @@ class PrepareDocuments(Action):
if not CONFIG.git_repo:
# Create and initialize the workspace folder, initialize the Git environment.
project_name = CONFIG.project_name or FileRepository.new_filename()
workdir = Path(CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name)
workdir = CONFIG.project_path
if not workdir and CONFIG.workspace:
workdir = Path(CONFIG.workspace) / project_name
workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name)
if not CONFIG.inc and workdir.exists():
shutil.rmtree(workdir)
CONFIG.git_repo = GitRepository()

10
metagpt/actions/project_management.py
View file

@ -183,6 +183,10 @@ MERGE_PROMPT = """
## Old Tasks
{old_tasks}
-----
## Format example
{format_example}
-----
Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules.
Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
@ -201,7 +205,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format,
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format,
and only output the json inside this tag, nothing else
"""
@ -264,7 +268,9 @@ class WriteTasks(Action):
return rsp
async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document:
prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content)
_, format_example = get_template(templates, format)
prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content,
format_example=format_example)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
task_doc.content = rsp.instruct_content.json(ensure_ascii=False)
return task_doc

9
metagpt/actions/summarize_code.py
View file

@ -3,12 +3,15 @@
"""
@Author : alexanderwu
@File : summarize_code.py
@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode.
"""
from pathlib import Path
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.logs import logger
from metagpt.utils.file_repository import FileRepository
@ -95,8 +98,10 @@ class SummarizeCode(Action):
return code_rsp
async def run(self):
design_doc = await FileRepository.get_file(self.context.design_filename)
task_doc = await FileRepository.get_file(self.context.task_filename)
design_pathname = Path(self.context.design_filename)
design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO)
task_pathname = Path(self.context.task_filename)
task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO)
src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace)
code_blocks = []
for filename in self.context.codes_filenames:

20
metagpt/actions/write_code.py
View file

@ -15,13 +15,13 @@
RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
"""
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.const import TEST_OUTPUTS_FILE_REPO
from metagpt.config import CONFIG
from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import CodingContext, RunCodeResult
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
@ -50,6 +50,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
# Debug logs
```text
{logs}
{summary_log}
```
-----
@ -90,18 +92,26 @@ class WriteCode(Action):
test_doc = await FileRepository.get_file(
filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO
)
summary_doc = None
if coding_context.design_doc.filename:
summary_doc = await FileRepository.get_file(
filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO
)
logs = ""
if test_doc:
test_detail = RunCodeResult.loads(test_doc.content)
logs = test_detail.stderr
prompt = PROMPT_TEMPLATE.format(
design=coding_context.design_doc.content,
tasks=coding_context.task_doc.content,
code=coding_context.code_doc.content,
tasks=coding_context.task_doc.content if coding_context.task_doc else "",
code=coding_context.code_doc.content if coding_context.code_doc else "",
logs=logs,
filename=self.context.filename,
summary_log=summary_doc.content if summary_doc else "",
)
logger.info(f"Writing {coding_context.filename}..")
code = await self.write_code(prompt)
if not coding_context.code_doc:
coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace)
coding_context.code_doc.content = code
return coding_context

3
metagpt/actions/write_code_review.py
View file

@ -108,10 +108,11 @@ class WriteCodeReview(Action):
k = CONFIG.code_review_k_times or 1
for i in range(k):
format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename)
task_content = self.context.task_doc.content if self.context.task_doc else ""
context = "\n----------\n".join(
[
"```text\n" + self.context.design_doc.content + "```\n",
"```text\n" + self.context.task_doc.content + "```\n",
"```text\n" + task_content + "```\n",
"```python\n" + self.context.code_doc.content + "```\n",
]
)

48
metagpt/actions/write_prd.py
View file

@ -8,6 +8,7 @@
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality.
3. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
"""
from __future__ import annotations
@ -27,6 +28,7 @@ from metagpt.const import (
)
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.get_template import get_template
from metagpt.utils.mermaid import mermaid_to_file
@ -53,7 +55,7 @@ ATTENTION: Output carefully referenced "Format example" in format.
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
"Original Requirements": "", # str, place the polished complete original requirements here
"project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc.
"Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc.
"Search Information": "",
"Requirements": "",
"Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals.
@ -85,9 +87,10 @@ and only output the json inside this tag, nothing else
""",
"FORMAT_EXAMPLE": """
[CONTENT]
{
{{
"Language": "",
"Original Requirements": "",
"Project Name": "{project_name}",
"Search Information": "",
"Requirements": "",
"Product Goals": [],
@ -111,7 +114,7 @@ and only output the json inside this tag, nothing else
"Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]],
"UI Design draft": "",
"Anything UNCLEAR": "",
}
}}
[/CONTENT]
""",
},
@ -228,6 +231,7 @@ There are no unclear points.
OUTPUT_MAPPING = {
"Language": (str, ...),
"Original Requirements": (str, ...),
"Project Name": (str, ...),
"Product Goals": (List[str], ...),
"User Stories": (List[str], ...),
"Competitive Analysis": (List[str], ...),
@ -270,7 +274,7 @@ ATTENTION: Output carefully referenced "Old PRD" in format.
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
"Original Requirements": "", # str, place the polished complete original requirements here
"project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc.
"Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc.
"Search Information": "",
"Requirements": "",
"Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals.
@ -320,6 +324,7 @@ class WritePRD(Action):
if not prd_doc:
continue
change_files.docs[prd_doc.filename] = prd_doc
logger.info(f"REWRITE PRD:{prd_doc.filename}")
# If there is no existing PRD, generate one using 'docs/requirement.txt'.
if not change_files.docs:
prd_doc = await self._update_prd(
@ -327,6 +332,7 @@ class WritePRD(Action):
)
if prd_doc:
change_files.docs[prd_doc.filename] = prd_doc
logger.info(f"NEW PRD:{prd_doc.filename}")
# Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the
# 'publish' message to transition the workflow to the next stage. This design allows room for global
# optimization in subsequent steps.
@ -343,32 +349,36 @@ class WritePRD(Action):
# logger.info(format)
prompt_template, format_example = get_template(templates, format)
project_name = CONFIG.project_name if CONFIG.project_name else ""
format_example = format_example.format(project_name=project_name)
# logger.info(prompt_template)
# logger.info(format_example)
prompt = prompt_template.format(
requirements=requirements, search_information=info, format_example=format_example
requirements=requirements, search_information=info, format_example=format_example, project_name=project_name
)
# logger.info(prompt)
# prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
await self._rename_workspace(prd)
return prd
async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool:
m = json.loads(old_prd_doc.content)
if m.get("Original Requirements") == new_requirement_doc.content:
# There have been no changes in the requirements, so they are considered unrelated.
return False
prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content)
res = await self._aask(prompt=prompt)
logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}")
logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}")
if "YES" in res:
return True
return False
async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document:
prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content)
if not CONFIG.project_name:
CONFIG.project_name = Path(CONFIG.project_path).name
prompt = MERGE_PROMPT.format(
requirements=new_requirement_doc.content, old_prd=prd_doc.content, project_name=CONFIG.project_name
)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
prd_doc.content = prd.instruct_content.json(ensure_ascii=False)
await self._rename_workspace(prd)
return prd_doc
async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None:
@ -404,3 +414,19 @@ class WritePRD(Action):
@staticmethod
async def _save_pdf(prd_doc):
await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO)
@staticmethod
async def _rename_workspace(prd):
if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to
# Section 2.2.3.10 of RFC 135
if not CONFIG.project_name:
CONFIG.project_name = Path(CONFIG.project_path).name
return
if not CONFIG.project_name:
if isinstance(prd, ActionOutput):
ws_name = prd.instruct_content.dict()["Project Name"]
else:
ws_name = CodeParser.parse_str(block="Project Name", text=prd)
CONFIG.project_name = ws_name
CONFIG.git_repo.rename_root(CONFIG.project_name)

7
metagpt/actions/write_test.py
View file

@ -9,8 +9,9 @@
"""
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import TEST_CODES_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import TestingContext
from metagpt.schema import Document, TestingContext
from metagpt.utils.common import CodeParser
PROMPT_TEMPLATE = """
@ -52,6 +53,10 @@ class WriteTest(Action):
return code
async def run(self, *args, **kwargs) -> TestingContext:
if not self.context.test_doc:
self.context.test_doc = Document(
filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO
)
prompt = PROMPT_TEMPLATE.format(
code_to_test=self.context.code_doc.content,
test_file_name=self.context.test_doc.filename,

3
metagpt/const.py
View file

@ -7,6 +7,7 @@
@Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for
common properties in the Message.
@Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135.
@Modified By: mashenquan, 2023/12/5. Add directories for code summarization..
"""
import contextvars
import os
@ -87,5 +88,7 @@ PRD_PDF_FILE_REPO = "resources/prd"
TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks"
TEST_CODES_FILE_REPO = "tests"
TEST_OUTPUTS_FILE_REPO = "test_outputs"
CODE_SUMMARIES_FILE_REPO = "docs/code_summaries"
CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries"
YAPI_URL = "http://yapi.deepwisdomai.com/"

4
metagpt/provider/base_gpt_api.py
View file

@ -38,7 +38,7 @@ class BaseGPTAPI(BaseChatbot):
rsp = self.completion(message)
return self.get_choice_text(rsp)
async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str:
async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, stream=True) -> str:
if system_msgs:
message = (
self._system_msgs(system_msgs) + [self._user_msg(msg)]
@ -49,7 +49,7 @@ class BaseGPTAPI(BaseChatbot):
message = (
[self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)]
)
rsp = await self.acompletion_text(message, stream=True)
rsp = await self.acompletion_text(message, stream=stream)
logger.debug(message)
# logger.debug(rsp)
return rsp

134
metagpt/roles/engineer.py
View file

@ -13,17 +13,25 @@
@Modified By: mashenquan, 2023-11-27.
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality.
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
from __future__ import annotations
import json
from collections import defaultdict
from pathlib import Path
from typing import Set
from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.config import CONFIG
from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.const import (
CODE_SUMMARIES_FILE_REPO,
CODE_SUMMARIES_PDF_FILE_REPO,
SYSTEM_DESIGN_FILE_REPO,
TASK_FILE_REPO,
)
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import (
@ -33,6 +41,16 @@ from metagpt.schema import (
Documents,
Message,
)
from metagpt.utils.common import any_to_str, any_to_str_set
IS_PASS_PROMPT = """
{context}
----
Does the above log indicate anything that needs to be done?
If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format;
otherwise, answer 'YES' in JSON format.
"""
class Engineer(Role):
@ -60,7 +78,7 @@ class Engineer(Role):
"""Initializes the Engineer role with given attributes."""
super().__init__(name, profile, goal, constraints)
self.use_code_review = use_code_review
self._watch([WriteTasks])
self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview])
self.code_todos = []
self.summarize_todos = []
self.n_borg = n_borg
@ -105,39 +123,88 @@ class Engineer(Role):
if self._rc.todo is None:
return None
if isinstance(self._rc.todo, WriteCode):
changed_files = await self._act_sp_with_cr(review=self.use_code_review)
# Unit tests only.
if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files:
changed_files.add(CONFIG.REQA_FILENAME)
return Message(
content="\n".join(changed_files),
role=self.profile,
cause_by=WriteCodeReview if self.use_code_review else WriteCode,
send_to="Edward", # The name of QaEngineer
)
return await self._act_write_code()
if isinstance(self._rc.todo, SummarizeCode):
summaries = []
for todo in self.summarize_todos:
summary = await todo.run()
summaries.append(summary.json(ensure_ascii=False))
return await self._act_summarize()
return None
async def _act_write_code(self):
changed_files = await self._act_sp_with_cr(review=self.use_code_review)
return Message(
content="\n".join(changed_files),
role=self.profile,
cause_by=WriteCodeReview if self.use_code_review else WriteCode,
send_to=self,
sent_from=self,
)
async def _act_summarize(self):
code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO)
code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO)
tasks = []
src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir)
for todo in self.summarize_todos:
summary = await todo.run()
summary_filename = Path(todo.context.design_filename).with_suffix(".md").name
dependencies = {todo.context.design_filename, todo.context.task_filename}
for filename in todo.context.codes_filenames:
rpath = src_relative_path / filename
dependencies.add(str(rpath))
await code_summaries_pdf_file_repo.save(
filename=summary_filename, content=summary, dependencies=dependencies
)
is_pass, reason = await self._is_pass(summary)
if not is_pass:
todo.context.reason = reason
tasks.append(todo.context.dict())
await code_summaries_file_repo.save(
filename=Path(todo.context.design_filename).name,
content=todo.context.json(),
dependencies=dependencies,
)
else:
await code_summaries_file_repo.delete(filename=Path(todo.context.design_filename).name)
logger.info(f"--max-auto-summarize-code={CONFIG.max_auto_summarize_code}")
if not tasks or CONFIG.max_auto_summarize_code == 0:
return Message(
content="\n".join(summaries),
content="",
role=self.profile,
cause_by=SummarizeCode,
send_to=MESSAGE_ROUTE_TO_NONE,
sent_from=self,
send_to="Edward", # The name of QaEngineer
)
return None
# The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited.
# This parameter is used for debugging the workflow.
CONFIG.max_auto_summarize_code -= 1 if CONFIG.max_auto_summarize_code > 0 else 0
return Message(
content=json.dumps(tasks), role=self.profile, cause_by=SummarizeCode, send_to=self, sent_from=self
)
async def _is_pass(self, summary) -> (str, str):
rsp = await self._llm.aask(msg=IS_PASS_PROMPT.format(context=summary), stream=False)
logger.info(rsp)
if "YES" in rsp:
return True, rsp
return False, rsp
async def _think(self) -> Action | None:
if not CONFIG.src_workspace:
CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name
if not self.code_todos:
await self._new_code_actions()
elif not self.summarize_todos:
await self._new_summarize_actions()
else:
write_code_filters = any_to_str_set([WriteTasks, SummarizeCode])
summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview])
if not self._rc.news:
return None
return self._rc.todo # For agent store
msg = self._rc.news[0]
if msg.cause_by in write_code_filters:
logger.info(f"TODO WriteCode:{msg.json()}")
await self._new_code_actions()
return self._rc.todo
if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self):
logger.info(f"TODO SummarizeCode:{msg.json()}")
await self._new_summarize_actions()
return self._rc.todo
return None
@staticmethod
async def _new_coding_context(
@ -151,9 +218,9 @@ class Engineer(Role):
design_doc = None
for i in dependencies:
if str(i.parent) == TASK_FILE_REPO:
task_doc = task_file_repo.get(i.filename)
task_doc = await task_file_repo.get(i.name)
elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO:
design_doc = design_file_repo.get(i.filename)
design_doc = await design_file_repo.get(i.name)
context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc)
return context
@ -216,16 +283,13 @@ class Engineer(Role):
async def _new_summarize_actions(self):
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
changed_src_files = src_file_repo.changed_files
src_files = src_file_repo.all_files
# Generate a SummarizeCode action for each pair of (system_design_doc, task_doc).
summarizations = {}
for filename in changed_src_files:
dependencies = src_file_repo.get_dependency(filename=filename)
summarizations = defaultdict(list)
for filename in src_files:
dependencies = await src_file_repo.get_dependency(filename=filename)
ctx = CodeSummarizeContext.loads(filenames=dependencies)
if ctx not in summarizations:
summarizations[ctx] = set()
srcs = summarizations.get(ctx)
srcs.add(filename)
summarizations[ctx].append(filename)
for ctx, filenames in summarizations.items():
ctx.codes_filenames = filenames
self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm))

12
metagpt/roles/qa_engineer.py
View file

@ -11,10 +11,13 @@
WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function.
2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message
to using file references.
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest
# from metagpt.const import WORKSPACE_ROOT
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.config import CONFIG
from metagpt.const import (
MESSAGE_ROUTE_TO_NONE,
@ -40,13 +43,16 @@ class QaEngineer(Role):
self._init_actions(
[WriteTest]
) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates
self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError])
self._watch([SummarizeCode, WriteTest, RunCode, DebugError])
self.test_round = 0
self.test_round_allowed = test_round_allowed
async def _write_test(self, message: Message) -> None:
changed_files = message.content.splitlines()
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
changed_files = set(src_file_repo.changed_files.keys())
# Unit tests only.
if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files:
changed_files.add(CONFIG.reqa_file)
tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO)
for filename in changed_files:
# write tests
@ -146,7 +152,7 @@ class QaEngineer(Role):
)
return result_msg
code_filters = any_to_str_set({WriteCode, WriteCodeReview})
code_filters = any_to_str_set({SummarizeCode})
test_filters = any_to_str_set({WriteTest, DebugError})
run_filters = any_to_str_set({RunCode})
for msg in self._rc.news:

3
metagpt/roles/role.py
View file

@ -284,9 +284,10 @@ class Role:
instruct_content=response.instruct_content,
role=self.profile,
cause_by=self._rc.todo,
sent_from=self,
)
else:
msg = Message(content=response, role=self.profile, cause_by=self._rc.todo)
msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self)
self._rc.memory.add(msg)
return msg

20
metagpt/schema.py
View file

@ -48,9 +48,9 @@ class Document(BaseModel):
Represents a document.
"""
root_path: str
filename: str
content: Optional[str] = None
root_path: str = ""
filename: str = ""
content: str = ""
def get_meta(self) -> Document:
"""Get metadata of the document.
@ -260,8 +260,8 @@ class MessageQueue:
class CodingContext(BaseModel):
filename: str
design_doc: Document
task_doc: Document
code_doc: Document
task_doc: Optional[Document]
code_doc: Optional[Document]
@staticmethod
def loads(val: str) -> CodingContext | None:
@ -275,7 +275,7 @@ class CodingContext(BaseModel):
class TestingContext(BaseModel):
filename: str
code_doc: Document
test_doc: Document
test_doc: Optional[Document]
@staticmethod
def loads(val: str) -> TestingContext | None:
@ -324,10 +324,11 @@ class RunCodeResult(BaseModel):
class CodeSummarizeContext(BaseModel):
design_filename: str = ""
task_filename: str = ""
codes_filenames: Set[str] = Field(default_factory=set)
codes_filenames: List[str] = Field(default_factory=list)
reason: str = ""
@staticmethod
def loads(filenames: Set) -> CodeSummarizeContext:
def loads(filenames: List) -> CodeSummarizeContext:
ctx = CodeSummarizeContext()
for filename in filenames:
if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO):
@ -337,3 +338,6 @@ class CodeSummarizeContext(BaseModel):
ctx.task_filename = str(filename)
continue
return ctx
def __hash__(self):
return hash((self.design_filename, self.task_filename))

11
metagpt/startup.py
View file

@ -1,6 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import asyncio
from pathlib import Path
import typer
@ -24,6 +25,10 @@ def startup(
help="Specify the directory path of the old version project to fulfill the " "incremental requirements.",
),
reqa_file: str = typer.Option(default="", help="Specify the source file name for rewriting the quality test code."),
max_auto_summarize_code: int = typer.Option(
default=-1,
help="The maximum number of times the 'SummarizeCode' action is automatically invoked, with -1 indicating unlimited. This parameter is used for debugging the workflow.",
),
):
"""Run a startup. Be a boss."""
from metagpt.roles import (
@ -36,10 +41,14 @@ def startup(
from metagpt.team import Team
# Use in the PrepareDocuments action according to Section 2.2.3.5.1 of RFC 135.
CONFIG.project_path = project_path
if project_path:
inc = True
project_name = project_name or Path(project_path).name
CONFIG.project_name = project_name
CONFIG.inc = inc
CONFIG.project_path = project_path
CONFIG.reqa_file = reqa_file
CONFIG.max_auto_summarize_code = max_auto_summarize_code
company = Team()
company.hire(

5
metagpt/utils/dependency_file.py
View file

@ -14,6 +14,7 @@ from typing import Set
import aiofiles
from metagpt.config import CONFIG
from metagpt.logs import logger
@ -81,7 +82,7 @@ class DependencyFile:
if persist:
await self.save()
async def get(self, filename: Path | str, persist=False):
async def get(self, filename: Path | str, persist=True):
"""Get dependencies for a file asynchronously.
:param filename: The filename or path.
@ -91,7 +92,7 @@ class DependencyFile:
if persist:
await self.load()
root = self._filename.parent
root = CONFIG.git_repo.workdir
try:
key = Path(filename).relative_to(root)
except ValueError:

33
metagpt/utils/file_repository.py
View file

@ -151,6 +151,17 @@ class FileRepository:
relative_files[str(rf)] = ct
return relative_files
@property
def all_files(self) -> List:
"""Get a dictionary of all files in the repository.
The dictionary includes file paths relative to the current FileRepository.
:return: A dictionary where keys are file paths and values are file information.
:rtype: List
"""
return self._git_repo.get_files(relative_path=self._relative_path)
def get_change_dir_files(self, dir: Path | str) -> List:
"""Get the files in a directory that have changed.
@ -259,3 +270,25 @@ class FileRepository:
"""
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies)
async def delete(self, filename: Path | str):
"""Delete a file from the file repository.
This method deletes a file from the file repository based on the provided filename.
:param filename: The name or path of the file to be deleted.
:type filename: Path or str
"""
pathname = self.workdir / filename
if not pathname.exists():
return
pathname.unlink(missing_ok=True)
dependency_file = await self._git_repo.get_dependency()
await dependency_file.update(filename=pathname, dependencies=None)
logger.info(f"remove dependency key: {str(pathname)}")
@staticmethod
async def delete_file(filename: Path | str, relative_path: Path | str = "."):
file_repo = CONFIG.git_repo.new_file_repository(relative_path=relative_path)
await file_repo.delete(filename=filename)

41
metagpt/utils/git_repository.py
View file

@ -8,10 +8,11 @@
"""
from __future__ import annotations
import os
import shutil
from enum import Enum
from pathlib import Path
from typing import Dict
from typing import Dict, List
from git.repo import Repo
from git.repo.fun import is_git_dir
@ -196,10 +197,46 @@ class GitRepository:
if new_path.exists():
logger.info(f"Delete directory {str(new_path)}")
shutil.rmtree(new_path)
self.workdir.rename(new_path)
try:
shutil.move(src=str(self.workdir), dst=str(new_path))
except Exception as e:
logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}")
logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}")
self._repository = Repo(new_path)
def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None) -> List:
"""Retrieve a list of files in the specified relative path.
The method returns a list of file paths relative to the current FileRepository.
:param relative_path: The relative path within the repository.
:type relative_path: Path or str
:param root_relative_path: The root relative path within the repository.
:type root_relative_path: Path or str
:return: A list of file paths in the specified directory.
:rtype: List[str]
"""
try:
relative_path = Path(relative_path).relative_to(self.workdir)
except ValueError:
relative_path = Path(relative_path)
if not root_relative_path:
root_relative_path = Path(self.workdir) / relative_path
files = []
try:
directory_path = Path(self.workdir) / relative_path
for file_path in directory_path.iterdir():
if file_path.is_file():
rpath = file_path.relative_to(root_relative_path)
files.append(str(rpath))
else:
subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path)
files.extend(subfolder_files)
except Exception as e:
logger.error(f"Error: {e}")
return files
if __name__ == "__main__":
path = DEFAULT_WORKSPACE_ROOT / "git"