From 438fbe28c06aefee542bdd005941f801d4fe3e56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 23 Nov 2023 11:29:09 +0800 Subject: [PATCH] refactor: save files --- metagpt/actions/design_api.py | 11 ++---- metagpt/actions/project_management.py | 54 +++++++++++++++++---------- metagpt/actions/write_prd.py | 23 +++++++----- metagpt/const.py | 1 + metagpt/utils/file_repository.py | 18 +++++++-- 5 files changed, 67 insertions(+), 40 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 2c8c87558..a8f89473d 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -290,7 +290,7 @@ class WriteDesign(Action): data_api_design = m.get("Data structures and interface definitions") if not data_api_design: return - pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(DATA_API_DESIGN_FILE_REPO) / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") @@ -300,19 +300,16 @@ class WriteDesign(Action): seq_flow = m.get("Program call flow") if not seq_flow: return - pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc).with_suffix(".mmd") + pathname = CONFIG.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("") await WriteDesign._save_mermaid_file(seq_flow, pathname) logger.info(f"Saving sequence flow to {str(pathname)}") @staticmethod async def _save_pdf(design_doc): - m = json.loads(design_doc.content) file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_PDF_FILE_REPO) - await file_repo.save(filename=design_doc.filename, content=json_to_markdown(m)) - logger.info(f"Saving system design pdf to {design_doc.root_relative_path}") + await file_repo.save_pdf(doc=design_doc) @staticmethod async def _save_mermaid_file(data: str, pathname: Path): - if not pathname.parent.exists(): - pathname.parent.mkdir(parents=True, exists_ok=True) + pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(data, pathname) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 73481c780..686aa3689 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -11,7 +11,12 @@ from typing import List from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, WORKSPACE_ROOT +from metagpt.const import ( + SYSTEM_DESIGN_FILE_REPO, + TASK_FILE_REPO, + TASK_PDF_FILE_REPO, + WORKSPACE_ROOT, +) from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -190,46 +195,50 @@ class WriteTasks(Action): change_files = Documents() # 根据docs/system_designs/下的git head diff识别哪些task文档需要重写 for filename in changed_system_designs: - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) - if task_doc: - task_doc = await self._merge(system_design_doc, task_doc) - else: - rsp = await self._run(system_design_doc.content) - task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) - await self._update_requirements(task_doc) change_files.docs[filename] = task_doc # 根据docs/tasks/下的git head diff识别哪些task文件被用户修改了，需要重写 for filename in changed_tasks: if filename in change_files.docs: continue - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) - task_doc = await self._merge(system_design_doc, task_doc) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + task_doc = await self._update_tasks( + filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo ) - await self._update_requirements(task_doc) change_files.docs[filename] = task_doc # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) - async def _run(self, context, format=CONFIG.prompt_format): + async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): + system_design_doc = await system_design_file_repo.get(filename) + task_doc = await tasks_file_repo.get(filename) + if task_doc: + task_doc = await self._merge(system_design_doc=system_design_doc, task_dock=task_doc) + else: + rsp = await self._run_new_tasks(context=system_design_doc.content) + task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) + await tasks_file_repo.save( + filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} + ) + await self._update_requirements(task_doc) + await self._save_pdf(task_doc=task_doc) + return task_doc + + async def _run_new_tasks(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) - self._save(context, rsp) + # self._save(context, rsp) return rsp async def _merge(self, system_design_doc, task_dock) -> Document: return task_dock - async def _update_requirements(self, doc): + @staticmethod + async def _update_requirements(doc): m = json.loads(doc.content) packages = set(m.get("Required Python third-party packages", set())) file_repo = CONFIG.git_repo.new_file_repository() @@ -244,6 +253,11 @@ class WriteTasks(Action): packages.add(pkg) await file_repo.save(filename, content="\n".join(packages)) + @staticmethod + async def _save_pdf(task_doc): + file_repo = CONFIG.git_repo.new_file_repository(TASK_PDF_FILE_REPO) + await file_repo.save_pdf(doc=task_doc) + class AssignTasks(Action): async def run(self, *args, **kwargs): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 34001dec1..8b03ac29a 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -25,7 +25,6 @@ from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -245,13 +244,17 @@ class WritePRD(Action): prd_docs = await prds_file_repo.get_all() change_files = Documents() for prd_doc in prd_docs: - prd_doc = await self._update_prd(requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs + ) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd if not change_files.docs: - prd_doc = await self._update_prd(requirement_doc, None, prds_file_repo) + prd_doc = await self._update_prd( + requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs + ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 @@ -283,9 +286,7 @@ class WritePRD(Action): async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: - prd = await self._run_new_requirement( - requirements=[requirement_doc.content], format=format, *args, **kwargs - ) + prd = await self._run_new_requirement(requirements=[requirement_doc.content], *args, **kwargs) new_prd_doc = Document( root_path=PRDS_FILE_REPO, filename=FileRepository.new_file_name() + ".json", @@ -298,6 +299,7 @@ class WritePRD(Action): await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) await self._save_competitive_analysis(new_prd_doc) await self._save_pdf(new_prd_doc) + return new_prd_doc @staticmethod async def _save_competitive_analysis(prd_doc): @@ -305,13 +307,14 @@ class WritePRD(Action): quadrant_chart = m.get("Competitive Quadrant Chart") if not quadrant_chart: return - pathname = CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc).with_suffix(".mmd") + pathname = ( + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + ) if not pathname.parent.exists(): - pathname.parent.mkdir(parents=True, exists_ok=True) + pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(quadrant_chart, pathname) @staticmethod async def _save_pdf(prd_doc): - m = json.loads(prd_doc.content) file_repo = CONFIG.git_repo.new_file_repository(PRD_PDF_FILE_REPO) - await file_repo.save(filename=prd_doc.filename, content=json_to_markdown(m)) + await file_repo.save_pdf(doc=prd_doc) diff --git a/metagpt/const.py b/metagpt/const.py index b5ecad7cc..7ee06ff7d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -60,3 +60,4 @@ DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" SEQ_FLOW_FILE_REPO = "resources/seq_flow" SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" +TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index ee6811209..62ba99d42 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -8,8 +8,8 @@ """ from __future__ import annotations +import json import os -import uuid from datetime import datetime from pathlib import Path from typing import Dict, List, Set @@ -18,6 +18,7 @@ import aiofiles from metagpt.logs import logger from metagpt.schema import Document +from metagpt.utils.json_to_markdown import json_to_markdown class FileRepository: @@ -165,5 +166,16 @@ class FileRepository: :return: A new filename string. """ current_time = datetime.now().strftime("%Y%m%d%H%M%S") - guid_suffix = str(uuid.uuid4())[:8] - return f"{current_time}x{guid_suffix}" + return current_time + # guid_suffix = str(uuid.uuid4())[:8] + # return f"{current_time}x{guid_suffix}" + + async def save_pdf(self, doc: Document): + """Save a Document as a PDF file. + + :param doc: The Document instance to be saved. + """ + m = json.loads(doc.content) + filename = Path(doc.filename).with_suffix(".md") + await self.save(filename=str(filename), content=json_to_markdown(m)) + logger.info(f"File Saved: {str(filename)}")