From e8131652de02a93454343d059dec02199f27b459 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 21:45:44 +0800 Subject: [PATCH] refactor: write prd & system design --- metagpt/actions/design_api.py | 83 ++++++++++++++++++++++++++--------- metagpt/actions/write_prd.py | 70 ++++++++++++++++++++++------- metagpt/const.py | 5 +++ 3 files changed, 123 insertions(+), 35 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index e7ee87fa2..3bbde24ea 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -5,13 +5,21 @@ @Author : alexanderwu @File : design_api.py """ +import json import shutil from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import PRDS_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import ( + DATA_API_DESIGN_FILE_REPO, + PRDS_FILE_REPO, + SEQ_FLOW_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + SYSTEM_DESIGN_PDF_FILE_REPO, + WORKSPACE_ROOT, +) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser @@ -214,40 +222,29 @@ class WriteDesign(Action): # 对于那些发生变动的PRD和设计文档，重新生成设计内容； changed_files = Documents() for filename in changed_prds.keys(): - prd = await prds_file_repo.get(filename) - old_system_design_doc = await system_design_file_repo.get(filename) - if not old_system_design_doc: - system_design = await self._run(context=prd.content) - doc = Document( - root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() - ) - else: - doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=doc.content, dependencies={prd.root_relative_path} + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) changed_files.docs[filename] = doc for filename in changed_system_designs.keys(): if filename in changed_files.docs: continue - prd_doc = await prds_file_repo.get(filename=filename) - old_system_design_doc = await system_design_file_repo.get(filename) - new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=new_system_design_doc.content, dependencies={prd_doc.root_relative_path} + doc = await self._update_system_design( + filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) - changed_files.docs[filename] = new_system_design_doc + changed_files.docs[filename] = doc # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=changed_files.json(), instruct_content=changed_files) - async def _run(self, context, format=CONFIG.prompt_format): + async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python + # package name" contain space, have to use setattr setattr( system_design.instruct_content, "Python package name", @@ -268,3 +265,49 @@ class WriteDesign(Action): else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) CONFIG.git_repo.rename_root(ws_name) + + async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._new_system_design(context=prd.content) + doc = Document( + root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + ) + else: + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} + ) + await self._save_data_api_design(doc) + await self._save_seq_flow(doc) + await self._save_pdf(doc) + return doc + + @staticmethod + async def _save_data_api_design(design_doc): + m = json.loads(design_doc.content) + data_api_design = m.get("Data structures and interface definitions") + if not data_api_design: + return + path = CONFIG.git_repo.workdir / DATA_API_DESIGN_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(data_api_design, path / Path(design_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_seq_flow(design_doc): + m = json.loads(design_doc.content) + seq_flow = m.get("Program call flow") + if not seq_flow: + return + path = CONFIG.git_repo.workdir / SEQ_FLOW_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(seq_flow, path / Path(design_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_pdf(design_doc): + m = json.loads(design_doc.content) + file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) + await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index a16d1ec06..df35ec865 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -5,16 +5,28 @@ @Author : alexanderwu @File : write_prd.py """ +from __future__ import annotations + +import json +from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG -from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME +from metagpt.const import ( + COMPETITIVE_ANALYSIS_FILE_REPO, + DOCS_FILE_REPO, + PRD_PDF_FILE_REPO, + PRDS_FILE_REPO, + REQUIREMENT_FILENAME, +) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template +from metagpt.utils.json_to_markdown import json_to_markdown +from metagpt.utils.mermaid import mermaid_to_file templates = { "json": { @@ -233,22 +245,15 @@ class WritePRD(Action): prd_docs = await prds_file_repo.get_all() change_files = Documents() for prd_doc in prd_docs: - if await self._is_relative_to(requirement_doc, prd_doc): - prd_doc = await self._merge(requirement_doc, prd_doc) - await prds_file_repo.save(filename=prd_doc.filename, content=prd_doc.content) - change_files.docs[prd_doc.filename] = prd_doc + prd_doc = await self._update_prd(requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) + if not prd_doc: + continue + change_files.docs[prd_doc.filename] = prd_doc # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd if not change_files.docs: - prd = await self._run_new_requirement( - requirements=[requirement_doc.content], format=format, *args, **kwargs - ) - doc = Document( - root_path=PRDS_FILE_REPO, - filename=FileRepository.new_file_name() + ".json", - content=prd.instruct_content.json(), - ) - await prds_file_repo.save(filename=doc.filename, content=doc.content) - change_files.docs[doc.filename] = doc + prd_doc = await self._update_prd(requirement_doc, None, prds_file_repo) + if prd_doc: + change_files.docs[prd_doc.filename] = prd_doc # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) @@ -275,3 +280,38 @@ class WritePRD(Action): async def _merge(self, doc1, doc2) -> Document: pass + + async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: + if not prd_doc: + prd = await self._run_new_requirement( + requirements=[requirement_doc.content], format=format, *args, **kwargs + ) + new_prd_doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_file_name() + ".json", + content=prd.instruct_content.json(), + ) + elif await self._is_relative_to(requirement_doc, prd_doc): + new_prd_doc = await self._merge(requirement_doc, prd_doc) + else: + return None + await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) + await self._save_competitive_analysis(new_prd_doc) + await self._save_pdf(new_prd_doc) + + @staticmethod + async def _save_competitive_analysis(prd_doc): + m = json.loads(prd_doc.content) + quadrant_chart = m.get("Competitive Quadrant Chart") + if not quadrant_chart: + return + path = CONFIG.git_repo.workdir / COMPETITIVE_ANALYSIS_FILE_REPO + if not path.exists(): + path.mkdir(parents=True, exists_ok=True) + await mermaid_to_file(quadrant_chart, path / Path(prd_doc).with_suffix(".mmd")) + + @staticmethod + async def _save_pdf(prd_doc): + m = json.loads(prd_doc.content) + file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) + await file_repo.save(filename=prd_doc.filename, content=json_to_markdown(m)) diff --git a/metagpt/const.py b/metagpt/const.py index 63f39f4a8..b5ecad7cc 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -55,3 +55,8 @@ DOCS_FILE_REPO = "docs" PRDS_FILE_REPO = "docs/prds" SYSTEM_DESIGN_FILE_REPO = "docs/system_design" TASK_FILE_REPO = "docs/tasks" +COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis" +DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" +SEQ_FLOW_FILE_REPO = "resources/seq_flow" +SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" +PRD_PDF_FILE_REPO = "resources/prd"