feat: +SummarizeCode, refactor project_name

This commit is contained in:
莘权 马 2023-12-04 23:04:07 +08:00
parent 838b3cfcc8
commit 9d84c8f047
31 changed files with 671 additions and 245 deletions

View file

@ -7,6 +7,7 @@
@Modified By: mashenquan, 2023/11/27.
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
"""
import json
from pathlib import Path
@ -23,7 +24,6 @@ from metagpt.const import (
)
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.get_template import get_template
from metagpt.utils.mermaid import mermaid_to_file
@ -43,7 +43,7 @@ Requirement: Fill in the following missing information based on the context, eac
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks.
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## Project name: Constant text.
## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
@ -58,15 +58,15 @@ and only output the json inside this tag, nothing else
""",
"FORMAT_EXAMPLE": """
[CONTENT]
{
{{
"Implementation approach": "We will ...",
"project_name": "snake_game",
"Project name": "{project_name}",
"File list": ["main.py"],
"Data structures and interfaces": '
classDiagram
class Game{
class Game{{
+int score
}
}}
...
Game "1" -- "1" Food: has
',
@ -77,7 +77,7 @@ and only output the json inside this tag, nothing else
G->>M: end game
',
"Anything UNCLEAR": "The requirement is clear to me."
}
}}
[/CONTENT]
""",
},
@ -96,7 +96,7 @@ ATTENTION: Output carefully referenced "Format example" in format.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## Project name: Constant text.
## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
@ -112,9 +112,9 @@ ATTENTION: Output carefully referenced "Format example" in format.
## Implementation approach
We will ...
## project_name
## Project name
```python
"snake_game"
"{project_name}"
```
## File list
@ -151,7 +151,7 @@ The requirement is clear to me.
OUTPUT_MAPPING = {
"Implementation approach": (str, ...),
"project_name": (str, ...),
"Project name": (str, ...),
"File list": (List[str], ...),
"Data structures and interfaces": (str, ...),
"Program call flow": (str, ...),
@ -173,7 +173,7 @@ ATTENTION: Output carefully referenced "Old Design" in format.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## Project name: Constant text "{project_name}".
## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
@ -229,50 +229,21 @@ class WriteDesign(Action):
async def _new_system_design(self, context, format=CONFIG.prompt_format):
prompt_template, format_example = get_template(templates, format)
format_example = format_example.format(project_name=CONFIG.project_name)
prompt = prompt_template.format(context=context, format_example=format_example)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
self._rename_project_name(system_design=system_design)
await self._rename_workspace(system_design)
return system_design
async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):
prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content)
prompt = MERGE_PROMPT.format(
old_design=system_design_doc.content, context=prd_doc.content, project_name=CONFIG.project_name
)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
# fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python
# package name" contain space, have to use setattr
self._rename_project_name(system_design=system_design)
system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False)
return system_design_doc
@staticmethod
def _rename_project_name(system_design):
# fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name"
# contain space, have to use setattr
if CONFIG.project_name:
setattr(
system_design.instruct_content,
"project_name",
CONFIG.project_name,
)
return
setattr(
system_design.instruct_content,
"project_name",
system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'),
)
@staticmethod
async def _rename_workspace(system_design):
if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to
# Section 2.2.3.10 of RFC 135
return
if isinstance(system_design, ActionOutput):
ws_name = system_design.instruct_content.dict()["project_name"]
else:
ws_name = CodeParser.parse_str(block="project_name", text=system_design)
CONFIG.git_repo.rename_root(ws_name)
async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document:
prd = await prds_file_repo.get(filename)
old_system_design_doc = await system_design_file_repo.get(filename)
@ -296,10 +267,10 @@ class WriteDesign(Action):
@staticmethod
async def _save_data_api_design(design_doc):
m = json.loads(design_doc.content)
data_api_design = m.get("Data structures and interface definitions")
data_api_design = m.get("Data structures and interfaces")
if not data_api_design:
return
pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("")
pathname = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("")
await WriteDesign._save_mermaid_file(data_api_design, pathname)
logger.info(f"Save class view to {str(pathname)}")

View file

@ -3,7 +3,7 @@
"""
@Time : 2023/11/20
@Author : mashenquan
@File : git_repository.py
@File : prepare_documents.py
@Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt.
RFC 135 2.2.3.5.1.
"""
@ -26,7 +26,10 @@ class PrepareDocuments(Action):
if not CONFIG.git_repo:
# Create and initialize the workspace folder, initialize the Git environment.
project_name = CONFIG.project_name or FileRepository.new_filename()
workdir = Path(CONFIG.project_path or DEFAULT_WORKSPACE_ROOT / project_name)
workdir = CONFIG.project_path
if not workdir and CONFIG.workspace:
workdir = Path(CONFIG.workspace) / project_name
workdir = Path(workdir or DEFAULT_WORKSPACE_ROOT / project_name)
if not CONFIG.inc and workdir.exists():
shutil.rmtree(workdir)
CONFIG.git_repo = GitRepository()

View file

@ -183,6 +183,10 @@ MERGE_PROMPT = """
## Old Tasks
{old_tasks}
-----
## Format example
{format_example}
-----
Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules.
Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
@ -201,7 +205,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format,
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format,
and only output the json inside this tag, nothing else
"""
@ -264,7 +268,9 @@ class WriteTasks(Action):
return rsp
async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document:
prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content)
_, format_example = get_template(templates, format)
prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content,
format_example=format_example)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
task_doc.content = rsp.instruct_content.json(ensure_ascii=False)
return task_doc

View file

@ -3,12 +3,15 @@
"""
@Author : alexanderwu
@File : summarize_code.py
@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode.
"""
from pathlib import Path
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.logs import logger
from metagpt.utils.file_repository import FileRepository
@ -95,8 +98,10 @@ class SummarizeCode(Action):
return code_rsp
async def run(self):
design_doc = await FileRepository.get_file(self.context.design_filename)
task_doc = await FileRepository.get_file(self.context.task_filename)
design_pathname = Path(self.context.design_filename)
design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO)
task_pathname = Path(self.context.task_filename)
task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO)
src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace)
code_blocks = []
for filename in self.context.codes_filenames:

View file

@ -15,13 +15,13 @@
RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
"""
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.const import TEST_OUTPUTS_FILE_REPO
from metagpt.config import CONFIG
from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import CodingContext, RunCodeResult
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
@ -50,6 +50,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc
# Debug logs
```text
{logs}
{summary_log}
```
-----
@ -90,18 +92,26 @@ class WriteCode(Action):
test_doc = await FileRepository.get_file(
filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO
)
summary_doc = None
if coding_context.design_doc.filename:
summary_doc = await FileRepository.get_file(
filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO
)
logs = ""
if test_doc:
test_detail = RunCodeResult.loads(test_doc.content)
logs = test_detail.stderr
prompt = PROMPT_TEMPLATE.format(
design=coding_context.design_doc.content,
tasks=coding_context.task_doc.content,
code=coding_context.code_doc.content,
tasks=coding_context.task_doc.content if coding_context.task_doc else "",
code=coding_context.code_doc.content if coding_context.code_doc else "",
logs=logs,
filename=self.context.filename,
summary_log=summary_doc.content if summary_doc else "",
)
logger.info(f"Writing {coding_context.filename}..")
code = await self.write_code(prompt)
if not coding_context.code_doc:
coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace)
coding_context.code_doc.content = code
return coding_context

View file

@ -108,10 +108,11 @@ class WriteCodeReview(Action):
k = CONFIG.code_review_k_times or 1
for i in range(k):
format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename)
task_content = self.context.task_doc.content if self.context.task_doc else ""
context = "\n----------\n".join(
[
"```text\n" + self.context.design_doc.content + "```\n",
"```text\n" + self.context.task_doc.content + "```\n",
"```text\n" + task_content + "```\n",
"```python\n" + self.context.code_doc.content + "```\n",
]
)

View file

@ -8,6 +8,7 @@
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality.
3. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD.
"""
from __future__ import annotations
@ -27,6 +28,7 @@ from metagpt.const import (
)
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.common import CodeParser
from metagpt.utils.file_repository import FileRepository
from metagpt.utils.get_template import get_template
from metagpt.utils.mermaid import mermaid_to_file
@ -53,7 +55,7 @@ ATTENTION: Output carefully referenced "Format example" in format.
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
"Original Requirements": "", # str, place the polished complete original requirements here
"project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc.
"Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc.
"Search Information": "",
"Requirements": "",
"Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals.
@ -85,9 +87,10 @@ and only output the json inside this tag, nothing else
""",
"FORMAT_EXAMPLE": """
[CONTENT]
{
{{
"Language": "",
"Original Requirements": "",
"Project Name": "{project_name}",
"Search Information": "",
"Requirements": "",
"Product Goals": [],
@ -111,7 +114,7 @@ and only output the json inside this tag, nothing else
"Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]],
"UI Design draft": "",
"Anything UNCLEAR": "",
}
}}
[/CONTENT]
""",
},
@ -228,6 +231,7 @@ There are no unclear points.
OUTPUT_MAPPING = {
"Language": (str, ...),
"Original Requirements": (str, ...),
"Project Name": (str, ...),
"Product Goals": (List[str], ...),
"User Stories": (List[str], ...),
"Competitive Analysis": (List[str], ...),
@ -270,7 +274,7 @@ ATTENTION: Output carefully referenced "Old PRD" in format.
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
"Original Requirements": "", # str, place the polished complete original requirements here
"project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc.
"Project Name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc.
"Search Information": "",
"Requirements": "",
"Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals.
@ -320,6 +324,7 @@ class WritePRD(Action):
if not prd_doc:
continue
change_files.docs[prd_doc.filename] = prd_doc
logger.info(f"REWRITE PRD:{prd_doc.filename}")
# If there is no existing PRD, generate one using 'docs/requirement.txt'.
if not change_files.docs:
prd_doc = await self._update_prd(
@ -327,6 +332,7 @@ class WritePRD(Action):
)
if prd_doc:
change_files.docs[prd_doc.filename] = prd_doc
logger.info(f"NEW PRD:{prd_doc.filename}")
# Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the
# 'publish' message to transition the workflow to the next stage. This design allows room for global
# optimization in subsequent steps.
@ -343,32 +349,36 @@ class WritePRD(Action):
# logger.info(format)
prompt_template, format_example = get_template(templates, format)
project_name = CONFIG.project_name if CONFIG.project_name else ""
format_example = format_example.format(project_name=project_name)
# logger.info(prompt_template)
# logger.info(format_example)
prompt = prompt_template.format(
requirements=requirements, search_information=info, format_example=format_example
requirements=requirements, search_information=info, format_example=format_example, project_name=project_name
)
# logger.info(prompt)
# prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
await self._rename_workspace(prd)
return prd
async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool:
m = json.loads(old_prd_doc.content)
if m.get("Original Requirements") == new_requirement_doc.content:
# There have been no changes in the requirements, so they are considered unrelated.
return False
prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content)
res = await self._aask(prompt=prompt)
logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}")
logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}")
if "YES" in res:
return True
return False
async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document:
prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content)
if not CONFIG.project_name:
CONFIG.project_name = Path(CONFIG.project_path).name
prompt = MERGE_PROMPT.format(
requirements=new_requirement_doc.content, old_prd=prd_doc.content, project_name=CONFIG.project_name
)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
prd_doc.content = prd.instruct_content.json(ensure_ascii=False)
await self._rename_workspace(prd)
return prd_doc
async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None:
@ -404,3 +414,19 @@ class WritePRD(Action):
@staticmethod
async def _save_pdf(prd_doc):
await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO)
@staticmethod
async def _rename_workspace(prd):
if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to
# Section 2.2.3.10 of RFC 135
if not CONFIG.project_name:
CONFIG.project_name = Path(CONFIG.project_path).name
return
if not CONFIG.project_name:
if isinstance(prd, ActionOutput):
ws_name = prd.instruct_content.dict()["Project Name"]
else:
ws_name = CodeParser.parse_str(block="Project Name", text=prd)
CONFIG.project_name = ws_name
CONFIG.git_repo.rename_root(CONFIG.project_name)

View file

@ -9,8 +9,9 @@
"""
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import TEST_CODES_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import TestingContext
from metagpt.schema import Document, TestingContext
from metagpt.utils.common import CodeParser
PROMPT_TEMPLATE = """
@ -52,6 +53,10 @@ class WriteTest(Action):
return code
async def run(self, *args, **kwargs) -> TestingContext:
if not self.context.test_doc:
self.context.test_doc = Document(
filename="test_" + self.context.code_doc.filename, root_path=TEST_CODES_FILE_REPO
)
prompt = PROMPT_TEMPLATE.format(
code_to_test=self.context.code_doc.content,
test_file_name=self.context.test_doc.filename,