diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index a3f7163c3..f6e2868e9 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -21,7 +21,7 @@ from metagpt.schema import ( SerializationMixin, TestingContext, ) -from metagpt.utils.file_repository import FileRepository +from metagpt.utils.project_repo import ProjectRepo class Action(SerializationMixin, ContextMixin, BaseModel): @@ -34,16 +34,8 @@ class Action(SerializationMixin, ContextMixin, BaseModel): node: ActionNode = Field(default=None, exclude=True) @property - def git_repo(self): - return self.context.git_repo - - @property - def file_repo(self): - return FileRepository(self.context.git_repo) - - @property - def src_workspace(self): - return self.context.src_workspace + def project_repo(self): + return ProjectRepo(git_repo=self.context.git_repo) @property def prompt_schema(self): diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 983214662..f491fdd55 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -13,7 +13,6 @@ import re from pydantic import Field from metagpt.actions.action import Action -from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger from metagpt.schema import RunCodeContext, RunCodeResult from metagpt.utils.common import CodeParser @@ -50,9 +49,7 @@ class DebugError(Action): i_context: RunCodeContext = Field(default_factory=RunCodeContext) async def run(self, *args, **kwargs) -> str: - output_doc = await self.file_repo.get_file( - filename=self.i_context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO - ) + output_doc = await self.project_repo.test_outputs.get(filename=self.i_context.output_filename) if not output_doc: return "" output_detail = RunCodeResult.loads(output_doc.content) @@ -62,14 +59,12 @@ class DebugError(Action): return "" logger.info(f"Debug and rewrite {self.i_context.test_filename}") - code_doc = await self.file_repo.get_file( - filename=self.i_context.code_filename, relative_path=self.context.src_workspace + code_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get( + filename=self.i_context.code_filename ) if not code_doc: return "" - test_doc = await self.file_repo.get_file( - filename=self.i_context.test_filename, relative_path=TEST_CODES_FILE_REPO - ) + test_doc = await self.project_repo.tests.get(filename=self.i_context.test_filename) if not test_doc: return "" prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 5f973bb60..04c580226 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -15,13 +15,7 @@ from typing import Optional from metagpt.actions import Action, ActionOutput from metagpt.actions.design_api_an import DESIGN_API_NODE -from metagpt.const import ( - DATA_API_DESIGN_FILE_REPO, - PRDS_FILE_REPO, - SEQ_FLOW_FILE_REPO, - SYSTEM_DESIGN_FILE_REPO, - SYSTEM_DESIGN_PDF_FILE_REPO, -) +from metagpt.const import DATA_API_DESIGN_FILE_REPO, SEQ_FLOW_FILE_REPO from metagpt.logs import logger from metagpt.schema import Document, Documents, Message from metagpt.utils.mermaid import mermaid_to_file @@ -46,27 +40,21 @@ class WriteDesign(Action): async def run(self, with_messages: Message, schema: str = None): # Use `git status` to identify which PRD documents have been modified in the `docs/prds` directory. - prds_file_repo = self.git_repo.new_file_repository(PRDS_FILE_REPO) - changed_prds = prds_file_repo.changed_files + changed_prds = self.project_repo.docs.prd.changed_files # Use `git status` to identify which design documents in the `docs/system_designs` directory have undergone # changes. - system_design_file_repo = self.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) - changed_system_designs = system_design_file_repo.changed_files + changed_system_designs = self.project_repo.docs.system_design.changed_files # For those PRDs and design documents that have undergone changes, regenerate the design content. changed_files = Documents() for filename in changed_prds.keys(): - doc = await self._update_system_design( - filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo - ) + doc = await self._update_system_design(filename=filename) changed_files.docs[filename] = doc for filename in changed_system_designs.keys(): if filename in changed_files.docs: continue - doc = await self._update_system_design( - filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo - ) + doc = await self._update_system_design(filename=filename) changed_files.docs[filename] = doc if not changed_files.docs: logger.info("Nothing has changed.") @@ -84,24 +72,22 @@ class WriteDesign(Action): system_design_doc.content = node.instruct_content.model_dump_json() return system_design_doc - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: - prd = await prds_file_repo.get(filename) - old_system_design_doc = await system_design_file_repo.get(filename) + async def _update_system_design(self, filename) -> Document: + prd = await self.project_repo.docs.prd.get(filename) + old_system_design_doc = await self.project_repo.docs.system_design.get(filename) if not old_system_design_doc: system_design = await self._new_system_design(context=prd.content) - doc = Document( - root_path=SYSTEM_DESIGN_FILE_REPO, + doc = await self.project_repo.docs.system_design.save( filename=filename, content=system_design.instruct_content.model_dump_json(), + dependencies={prd.root_relative_path}, ) else: doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) - await system_design_file_repo.save( - filename=filename, content=doc.content, dependencies={prd.root_relative_path} - ) + await self.project_repo.docs.system_design.save_doc(doc=doc, dependencies={prd.root_relative_path}) await self._save_data_api_design(doc) await self._save_seq_flow(doc) - await self._save_pdf(doc) + await self.project_repo.resources.system_design.save_pdf(doc=doc) return doc async def _save_data_api_design(self, design_doc): @@ -109,7 +95,7 @@ class WriteDesign(Action): data_api_design = m.get("Data structures and interfaces") if not data_api_design: return - pathname = self.git_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("") + pathname = self.project_repo.workdir / DATA_API_DESIGN_FILE_REPO / Path(design_doc.filename).with_suffix("") await self._save_mermaid_file(data_api_design, pathname) logger.info(f"Save class view to {str(pathname)}") @@ -118,13 +104,10 @@ class WriteDesign(Action): seq_flow = m.get("Program call flow") if not seq_flow: return - pathname = self.git_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("") + pathname = self.project_repo.workdir / Path(SEQ_FLOW_FILE_REPO) / Path(design_doc.filename).with_suffix("") await self._save_mermaid_file(seq_flow, pathname) logger.info(f"Saving sequence flow to {str(pathname)}") - async def _save_pdf(self, design_doc): - await self.file_repo.save_as(doc=design_doc, with_suffix=".md", relative_path=SYSTEM_DESIGN_PDF_FILE_REPO) - async def _save_mermaid_file(self, data: str, pathname: Path): pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(self.config.mermaid_engine, data, pathname) diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index 8a9e78b2a..56c587cb3 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -12,8 +12,7 @@ from pathlib import Path from typing import Optional from metagpt.actions import Action, ActionOutput -from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME -from metagpt.schema import Document +from metagpt.const import REQUIREMENT_FILENAME from metagpt.utils.file_repository import FileRepository from metagpt.utils.git_repository import GitRepository @@ -38,7 +37,6 @@ class PrepareDocuments(Action): if path.exists() and not self.config.inc: shutil.rmtree(path) self.config.project_path = path - self.config.project_name = path.name self.context.git_repo = GitRepository(local_path=path, auto_init=True) async def run(self, with_messages, **kwargs): @@ -46,9 +44,7 @@ class PrepareDocuments(Action): self._init_repo() # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. - doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) - await self.file_repo.save_file(filename=REQUIREMENT_FILENAME, content=doc.content, relative_path=DOCS_FILE_REPO) - + doc = await self.project_repo.docs.save(filename=REQUIREMENT_FILENAME, content=with_messages[0].content) # Send a Message notification to the WritePRD action, instructing it to process requirements using # `docs/requirement.txt` and `docs/prds/`. return ActionOutput(content=doc.content, instruct_content=doc) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index bb8141a74..9ada629be 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -16,12 +16,7 @@ from typing import Optional from metagpt.actions import ActionOutput from metagpt.actions.action import Action from metagpt.actions.project_management_an import PM_NODE -from metagpt.const import ( - PACKAGE_REQUIREMENTS_FILENAME, - SYSTEM_DESIGN_FILE_REPO, - TASK_FILE_REPO, - TASK_PDF_FILE_REPO, -) +from metagpt.const import PACKAGE_REQUIREMENTS_FILENAME from metagpt.logs import logger from metagpt.schema import Document, Documents @@ -39,27 +34,20 @@ class WriteTasks(Action): i_context: Optional[str] = None async def run(self, with_messages): - system_design_file_repo = self.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) - changed_system_designs = system_design_file_repo.changed_files - - tasks_file_repo = self.git_repo.new_file_repository(TASK_FILE_REPO) - changed_tasks = tasks_file_repo.changed_files + changed_system_designs = self.project_repo.docs.system_design.changed_files + changed_tasks = self.project_repo.docs.task.changed_files change_files = Documents() # Rewrite the system designs that have undergone changes based on the git head diff under # `docs/system_designs/`. for filename in changed_system_designs: - task_doc = await self._update_tasks( - filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo - ) + task_doc = await self._update_tasks(filename=filename) change_files.docs[filename] = task_doc # Rewrite the task files that have undergone changes based on the git head diff under `docs/tasks/`. for filename in changed_tasks: if filename in change_files.docs: continue - task_doc = await self._update_tasks( - filename=filename, system_design_file_repo=system_design_file_repo, tasks_file_repo=tasks_file_repo - ) + task_doc = await self._update_tasks(filename=filename) change_files.docs[filename] = task_doc if not change_files.docs: @@ -68,21 +56,22 @@ class WriteTasks(Action): # global optimization in subsequent steps. return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files) - async def _update_tasks(self, filename, system_design_file_repo, tasks_file_repo): - system_design_doc = await system_design_file_repo.get(filename) - task_doc = await tasks_file_repo.get(filename) + async def _update_tasks(self, filename): + system_design_doc = await self.project_repo.docs.system_design.get(filename) + task_doc = await self.project_repo.docs.task.get(filename) if task_doc: task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc) + await self.project_repo.docs.task.save_doc( + doc=task_doc, dependencies={system_design_doc.root_relative_path} + ) else: rsp = await self._run_new_tasks(context=system_design_doc.content) - task_doc = Document( - root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.model_dump_json() + task_doc = await self.project_repo.docs.task.save( + filename=filename, + content=rsp.instruct_content.model_dump_json(), + dependencies={system_design_doc.root_relative_path}, ) - await tasks_file_repo.save( - filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path} - ) await self._update_requirements(task_doc) - await self._save_pdf(task_doc=task_doc) return task_doc async def _run_new_tasks(self, context): @@ -98,8 +87,7 @@ class WriteTasks(Action): async def _update_requirements(self, doc): m = json.loads(doc.content) packages = set(m.get("Required Python third-party packages", set())) - file_repo = self.git_repo.new_file_repository() - requirement_doc = await file_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME) + requirement_doc = await self.project_repo.get(filename=PACKAGE_REQUIREMENTS_FILENAME) if not requirement_doc: requirement_doc = Document(filename=PACKAGE_REQUIREMENTS_FILENAME, root_path=".", content="") lines = requirement_doc.content.splitlines() @@ -107,7 +95,4 @@ class WriteTasks(Action): if pkg == "": continue packages.add(pkg) - await file_repo.save(PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) - - async def _save_pdf(self, task_doc): - await self.file_repo.save_as(doc=task_doc, with_suffix=".md", relative_path=TASK_PDF_FILE_REPO) + await self.project_repo.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content="\n".join(packages)) diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index dde41d3c6..182561d59 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -11,7 +11,6 @@ from pydantic import Field from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action -from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodeSummarizeContext @@ -99,11 +98,10 @@ class SummarizeCode(Action): async def run(self): design_pathname = Path(self.i_context.design_filename) - repo = self.file_repo - design_doc = await repo.get_file(filename=design_pathname.name, relative_path=SYSTEM_DESIGN_FILE_REPO) + design_doc = await self.project_repo.docs.system_design.get(filename=design_pathname.name) task_pathname = Path(self.i_context.task_filename) - task_doc = await repo.get_file(filename=task_pathname.name, relative_path=TASK_FILE_REPO) - src_file_repo = self.git_repo.new_file_repository(relative_path=self.context.src_workspace) + task_doc = await self.project_repo.docs.task.get(filename=task_pathname.name) + src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs code_blocks = [] for filename in self.i_context.codes_filenames: code_doc = await src_file_repo.get(filename) diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 1b3dcf5f0..c0f1b1a93 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -21,13 +21,7 @@ from pydantic import Field from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action -from metagpt.const import ( - BUGFIX_FILENAME, - CODE_SUMMARIES_FILE_REPO, - DOCS_FILE_REPO, - TASK_FILE_REPO, - TEST_OUTPUTS_FILE_REPO, -) +from metagpt.const import BUGFIX_FILENAME from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser @@ -94,16 +88,12 @@ class WriteCode(Action): return code async def run(self, *args, **kwargs) -> CodingContext: - bug_feedback = await self.file_repo.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO) + bug_feedback = await self.project_repo.docs.get(filename=BUGFIX_FILENAME) coding_context = CodingContext.loads(self.i_context.content) - test_doc = await self.file_repo.get_file( - filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO - ) + test_doc = await self.project_repo.test_outputs.get(filename="test_" + coding_context.filename + ".json") summary_doc = None if coding_context.design_doc and coding_context.design_doc.filename: - summary_doc = await self.file_repo.get_file( - filename=coding_context.design_doc.filename, relative_path=CODE_SUMMARIES_FILE_REPO - ) + summary_doc = await self.project_repo.docs.code_summary.get(filename=coding_context.design_doc.filename) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) @@ -115,8 +105,7 @@ class WriteCode(Action): code_context = await self.get_codes( coding_context.task_doc, exclude=self.i_context.filename, - git_repo=self.git_repo, - src_workspace=self.context.src_workspace, + project_repo=self.project_repo.with_src_path(self.context.src_workspace), ) prompt = PROMPT_TEMPLATE.format( @@ -138,16 +127,15 @@ class WriteCode(Action): return coding_context @staticmethod - async def get_codes(task_doc, exclude, git_repo, src_workspace) -> str: + async def get_codes(task_doc, exclude, project_repo) -> str: if not task_doc: return "" if not task_doc.content: - file_repo = git_repo.new_file_repository() - task_doc.content = file_repo.get_file(filename=task_doc.filename, relative_path=TASK_FILE_REPO) + task_doc = project_repo.docs.task.get(filename=task_doc.filename) m = json.loads(task_doc.content) code_filenames = m.get("Task list", []) codes = [] - src_file_repo = git_repo.new_file_repository(relative_path=src_workspace) + src_file_repo = project_repo.srcs for filename in code_filenames: if filename == exclude: continue diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index b25f1ab69..21281dde1 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -143,8 +143,7 @@ class WriteCodeReview(Action): code_context = await WriteCode.get_codes( self.i_context.task_doc, exclude=self.i_context.filename, - git_repo=self.context.git_repo, - src_workspace=self.src_workspace, + project_repo=self.project_repo.with_src_path(self.context.src_workspace), ) context = "\n".join( [ diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index a838dea8e..38ac62536 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -29,9 +29,6 @@ from metagpt.actions.write_prd_an import ( from metagpt.const import ( BUGFIX_FILENAME, COMPETITIVE_ANALYSIS_FILE_REPO, - DOCS_FILE_REPO, - PRD_PDF_FILE_REPO, - PRDS_FILE_REPO, REQUIREMENT_FILENAME, ) from metagpt.logs import logger @@ -67,11 +64,10 @@ class WritePRD(Action): async def run(self, with_messages, *args, **kwargs) -> ActionOutput | Message: # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are # related to the PRD. If they are related, rewrite the PRD. - docs_file_repo = self.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO) - requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME) + requirement_doc = await self.project_repo.docs.get(filename=REQUIREMENT_FILENAME) if requirement_doc and await self._is_bugfix(requirement_doc.content): - await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) - await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") + await self.project_repo.docs.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) + await self.project_repo.docs.save(filename=REQUIREMENT_FILENAME, content="") bug_fix = BugFixContext(filename=BUGFIX_FILENAME) return Message( content=bug_fix.model_dump_json(), @@ -82,24 +78,19 @@ class WritePRD(Action): send_to="Alex", # the name of Engineer ) else: - await docs_file_repo.delete(filename=BUGFIX_FILENAME) + await self.project_repo.docs.delete(filename=BUGFIX_FILENAME) - prds_file_repo = self.git_repo.new_file_repository(PRDS_FILE_REPO) - prd_docs = await prds_file_repo.get_all() + prd_docs = await self.project_repo.docs.prd.get_all() change_files = Documents() for prd_doc in prd_docs: - prd_doc = await self._update_prd( - requirement_doc=requirement_doc, prd_doc=prd_doc, prds_file_repo=prds_file_repo, *args, **kwargs - ) + prd_doc = await self._update_prd(requirement_doc=requirement_doc, prd_doc=prd_doc, *args, **kwargs) if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc logger.info(f"rewrite prd: {prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: - prd_doc = await self._update_prd( - requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs - ) + prd_doc = await self._update_prd(requirement_doc=requirement_doc, *args, **kwargs) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc logger.debug(f"new prd: {prd_doc.filename}") @@ -109,13 +100,6 @@ class WritePRD(Action): return ActionOutput(content=change_files.model_dump_json(), instruct_content=change_files) async def _run_new_requirement(self, requirements) -> ActionOutput: - # sas = SearchAndSummarize() - # # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) - # rsp = "" - # info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" - # if sas.result: - # logger.info(sas.result) - # logger.info(rsp) project_name = self.project_name context = CONTEXT_TEMPLATE.format(requirements=requirements, project_name=project_name) exclude = [PROJECT_NAME.key] if project_name else [] @@ -137,23 +121,21 @@ class WritePRD(Action): await self._rename_workspace(node) return prd_doc - async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: + async def _update_prd(self, requirement_doc, prd_doc=None, *args, **kwargs) -> Document | None: if not prd_doc: prd = await self._run_new_requirement( requirements=[requirement_doc.content if requirement_doc else ""], *args, **kwargs ) - new_prd_doc = Document( - root_path=PRDS_FILE_REPO, - filename=FileRepository.new_filename() + ".json", - content=prd.instruct_content.model_dump_json(), + new_prd_doc = await self.project_repo.docs.prd.save( + filename=FileRepository.new_filename() + ".json", content=prd.instruct_content.model_dump_json() ) elif await self._is_relative(requirement_doc, prd_doc): new_prd_doc = await self._merge(requirement_doc, prd_doc) + self.project_repo.docs.prd.save_doc(doc=new_prd_doc) else: return None - await prds_file_repo.save(filename=new_prd_doc.filename, content=new_prd_doc.content) await self._save_competitive_analysis(new_prd_doc) - await self._save_pdf(new_prd_doc) + await self.project_repo.resources.prd.save_pdf(doc=new_prd_doc) return new_prd_doc async def _save_competitive_analysis(self, prd_doc): @@ -161,14 +143,13 @@ class WritePRD(Action): quadrant_chart = m.get("Competitive Quadrant Chart") if not quadrant_chart: return - pathname = self.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + pathname = ( + self.project_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + ) if not pathname.parent.exists(): pathname.parent.mkdir(parents=True, exist_ok=True) await mermaid_to_file(self.config.mermaid_engine, quadrant_chart, pathname) - async def _save_pdf(self, prd_doc): - await self.file_repo.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) - async def _rename_workspace(self, prd): if not self.project_name: if isinstance(prd, (ActionOutput, ActionNode)): @@ -177,11 +158,14 @@ class WritePRD(Action): ws_name = CodeParser.parse_str(block="Project Name", text=prd) if ws_name: self.project_name = ws_name - self.git_repo.rename_root(self.project_name) + self.project_repo.git_repo.rename_root(self.project_name) async def _is_bugfix(self, context) -> bool: - src_workspace_path = self.git_repo.workdir / self.git_repo.workdir.name - code_files = self.git_repo.get_files(relative_path=src_workspace_path) + git_workdir = self.project_repo.git_repo.workdir + src_workdir = git_workdir / git_workdir.name + if not src_workdir.exists(): + return False + code_files = self.project_repo.with_src_path(path=git_workdir / git_workdir.name).srcs.all_files if not code_files: return False node = await WP_ISSUE_TYPE_NODE.fill(context, self.llm) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index bc56ca813..20dcce181 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -27,12 +27,7 @@ from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.fix_bug import FixBug from metagpt.actions.summarize_code import SummarizeCode -from metagpt.const import ( - CODE_SUMMARIES_FILE_REPO, - CODE_SUMMARIES_PDF_FILE_REPO, - SYSTEM_DESIGN_FILE_REPO, - TASK_FILE_REPO, -) +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -97,7 +92,6 @@ class Engineer(Role): async def _act_sp_with_cr(self, review=False) -> Set[str]: changed_files = set() - src_file_repo = self.git_repo.new_file_repository(self.src_workspace) for todo in self.code_todos: """ # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): @@ -112,8 +106,8 @@ class Engineer(Role): action = WriteCodeReview(i_context=coding_context, context=self.context, llm=self.llm) self._init_action_system_message(action) coding_context = await action.run() - await src_file_repo.save( - coding_context.filename, + await self.project_repo.srcs.save( + filename=coding_context.filename, dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path}, content=coding_context.code_doc.content, ) @@ -153,31 +147,28 @@ class Engineer(Role): ) async def _act_summarize(self): - code_summaries_file_repo = self.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) - code_summaries_pdf_file_repo = self.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) tasks = [] - src_relative_path = self.src_workspace.relative_to(self.git_repo.workdir) for todo in self.summarize_todos: summary = await todo.run() summary_filename = Path(todo.i_context.design_filename).with_suffix(".md").name dependencies = {todo.i_context.design_filename, todo.i_context.task_filename} for filename in todo.i_context.codes_filenames: - rpath = src_relative_path / filename + rpath = self.project_repo.src_relative_path / filename dependencies.add(str(rpath)) - await code_summaries_pdf_file_repo.save( + await self.project_repo.resources.code_summary.save( filename=summary_filename, content=summary, dependencies=dependencies ) is_pass, reason = await self._is_pass(summary) if not is_pass: todo.i_context.reason = reason tasks.append(todo.i_context.dict()) - await code_summaries_file_repo.save( + await self.project_repo.docs.code_summary.save( filename=Path(todo.i_context.design_filename).name, content=todo.i_context.model_dump_json(), dependencies=dependencies, ) else: - await code_summaries_file_repo.delete(filename=Path(todo.i_context.design_filename).name) + await self.project_repo.docs.code_summary.delete(filename=Path(todo.i_context.design_filename).name) logger.info(f"--max-auto-summarize-code={self.config.max_auto_summarize_code}") if not tasks or self.config.max_auto_summarize_code == 0: @@ -220,60 +211,54 @@ class Engineer(Role): return self.rc.todo return None - @staticmethod - async def _new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency - ) -> CodingContext: - old_code_doc = await src_file_repo.get(filename) + async def _new_coding_context(self, filename, dependency) -> CodingContext: + old_code_doc = await self.project_repo.srcs.get(filename) if not old_code_doc: - old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") + old_code_doc = Document(root_path=str(self.project_repo.src_relative_path), filename=filename, content="") dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)} task_doc = None design_doc = None for i in dependencies: if str(i.parent) == TASK_FILE_REPO: - task_doc = await task_file_repo.get(i.name) + task_doc = await self.project_repo.docs.task.get(i.name) elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO: - design_doc = await design_file_repo.get(i.name) + design_doc = await self.project_repo.docs.system_design.get(i.name) if not task_doc or not design_doc: logger.error(f'Detected source code "{filename}" from an unknown origin.') raise ValueError(f'Detected source code "{filename}" from an unknown origin.') context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc) return context - @staticmethod - async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency): - context = await Engineer._new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency - ) + async def _new_coding_doc(self, filename, dependency): + context = await self._new_coding_context(filename, dependency) coding_doc = Document( - root_path=str(src_file_repo.root_path), filename=filename, content=context.model_dump_json() + root_path=str(self.project_repo.src_relative_path), filename=filename, content=context.model_dump_json() ) return coding_doc async def _new_code_actions(self, bug_fix=False): # Prepare file repos - src_file_repo = self.git_repo.new_file_repository(self.src_workspace) - changed_src_files = src_file_repo.all_files if bug_fix else src_file_repo.changed_files - task_file_repo = self.git_repo.new_file_repository(TASK_FILE_REPO) - changed_task_files = task_file_repo.changed_files - design_file_repo = self.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) - + changed_src_files = self.project_repo.srcs.all_files if bug_fix else self.project_repo.srcs.changed_files + changed_task_files = self.project_repo.docs.task.changed_files changed_files = Documents() # Recode caused by upstream changes. for filename in changed_task_files: - design_doc = await design_file_repo.get(filename) - task_doc = await task_file_repo.get(filename) + design_doc = await self.project_repo.docs.system_design.get(filename) + task_doc = await self.project_repo.docs.task.get(filename) task_list = self._parse_tasks(task_doc) for task_filename in task_list: - old_code_doc = await src_file_repo.get(task_filename) + old_code_doc = await self.project_repo.srcs.get(task_filename) if not old_code_doc: - old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=task_filename, content="") + old_code_doc = Document( + root_path=str(self.project_repo.src_relative_path), filename=task_filename, content="" + ) context = CodingContext( filename=task_filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc ) coding_doc = Document( - root_path=str(src_file_repo.root_path), filename=task_filename, content=context.model_dump_json() + root_path=str(self.project_repo.src_relative_path), + filename=task_filename, + content=context.model_dump_json(), ) if task_filename in changed_files.docs: logger.warning( @@ -289,13 +274,7 @@ class Engineer(Role): for filename in changed_src_files: if filename in changed_files.docs: continue - coding_doc = await self._new_coding_doc( - filename=filename, - src_file_repo=src_file_repo, - task_file_repo=task_file_repo, - design_file_repo=design_file_repo, - dependency=dependency, - ) + coding_doc = await self._new_coding_doc(filename=filename, dependency=dependency) changed_files.docs[filename] = coding_doc self.code_todos.append(WriteCode(i_context=coding_doc, context=self.context, llm=self.llm)) @@ -303,13 +282,12 @@ class Engineer(Role): self.set_todo(self.code_todos[0]) async def _new_summarize_actions(self): - src_file_repo = self.git_repo.new_file_repository(self.src_workspace) - src_files = src_file_repo.all_files + src_files = self.project_repo.srcs.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). summarizations = defaultdict(list) for filename in src_files: - dependencies = await src_file_repo.get_dependency(filename=filename) - ctx = CodeSummarizeContext.loads(filenames=dependencies) + dependencies = await self.project_repo.srcs.get_dependency(filename=filename) + ctx = CodeSummarizeContext.loads(filenames=list(dependencies)) summarizations[ctx].append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index cd043b551..949085fe9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -17,11 +17,7 @@ from metagpt.actions import DebugError, RunCode, WriteTest from metagpt.actions.summarize_code import SummarizeCode -from metagpt.const import ( - MESSAGE_ROUTE_TO_NONE, - TEST_CODES_FILE_REPO, - TEST_OUTPUTS_FILE_REPO, -) +from metagpt.const import MESSAGE_ROUTE_TO_NONE, TEST_CODES_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -48,37 +44,32 @@ class QaEngineer(Role): self.test_round = 0 async def _write_test(self, message: Message) -> None: - src_file_repo = self.context.git_repo.new_file_repository(self.context.src_workspace) + src_file_repo = self.project_repo.with_src_path(self.context.src_workspace).srcs changed_files = set(src_file_repo.changed_files.keys()) # Unit tests only. if self.config.reqa_file and self.config.reqa_file not in changed_files: changed_files.add(self.config.reqa_file) - tests_file_repo = self.context.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests if not filename or "test" in filename: continue code_doc = await src_file_repo.get(filename) - test_doc = await tests_file_repo.get("test_" + code_doc.filename) + test_doc = await self.project_repo.tests.get("test_" + code_doc.filename) if not test_doc: test_doc = Document( - root_path=str(tests_file_repo.root_path), filename="test_" + code_doc.filename, content="" + root_path=str(self.project_repo.tests.root_path), filename="test_" + code_doc.filename, content="" ) logger.info(f"Writing {test_doc.filename}..") context = TestingContext(filename=test_doc.filename, test_doc=test_doc, code_doc=code_doc) context = await WriteTest(i_context=context, context=self.context, llm=self.llm).run() - await tests_file_repo.save( - filename=context.test_doc.filename, - content=context.test_doc.content, - dependencies={context.code_doc.root_relative_path}, - ) + await self.project_repo.tests.save_doc(doc=test_doc, dependencies={context.code_doc.root_relative_path}) # prepare context for run tests in next round run_code_context = RunCodeContext( command=["python", context.test_doc.root_relative_path], code_filename=context.code_doc.filename, test_filename=context.test_doc.filename, - working_directory=str(self.context.git_repo.workdir), + working_directory=str(self.project_repo.workdir), additional_python_paths=[str(self.context.src_workspace)], ) self.publish_message( @@ -91,25 +82,23 @@ class QaEngineer(Role): ) ) - logger.info(f"Done {str(tests_file_repo.workdir)} generating.") + logger.info(f"Done {str(self.project_repo.tests.workdir)} generating.") async def _run_code(self, msg): run_code_context = RunCodeContext.loads(msg.content) - src_doc = await self.context.git_repo.new_file_repository(self.context.src_workspace).get( + src_doc = await self.project_repo.with_src_path(self.context.src_workspace).srcs.get( run_code_context.code_filename ) if not src_doc: return - test_doc = await self.context.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get( - run_code_context.test_filename - ) + test_doc = await self.project_repo.tests.get(run_code_context.test_filename) if not test_doc: return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content result = await RunCode(i_context=run_code_context, context=self.context, llm=self.llm).run() run_code_context.output_filename = run_code_context.test_filename + ".json" - await self.context.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).save( + await self.project_repo.test_outputs.save( filename=run_code_context.output_filename, content=result.model_dump_json(), dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, @@ -132,7 +121,7 @@ class QaEngineer(Role): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) code = await DebugError(i_context=run_code_context, context=self.context, llm=self.llm).run() - await self.context.file_repo.save_file( + await self.project_repo.tests.save( filename=run_code_context.test_filename, content=code, relative_path=TEST_CODES_FILE_REPO ) run_code_context.output = None diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index e467ef83e..0ca353398 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -36,6 +36,7 @@ from metagpt.memory import Memory from metagpt.provider import HumanProvider from metagpt.schema import Message, MessageQueue, SerializationMixin from metagpt.utils.common import any_to_name, any_to_str, role_raise_decorator +from metagpt.utils.project_repo import ProjectRepo from metagpt.utils.repair_llm_raw_output import extract_state_value_from_output PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}. """ @@ -188,6 +189,11 @@ class Role(SerializationMixin, ContextMixin, BaseModel): def src_workspace(self, value): self.context.src_workspace = value + @property + def project_repo(self) -> ProjectRepo: + project_repo = ProjectRepo(git_repo=self.context.git_repo) + return project_repo.with_src_path(self.context.src_workspace) if self.context.src_workspace else project_repo + @property def prompt_schema(self): """Prompt schema: json/markdown""" @@ -427,7 +433,7 @@ class Role(SerializationMixin, ContextMixin, BaseModel): break # act logger.debug(f"{self._setting}: {self.rc.state=}, will do {self.rc.todo}") - rsp = await self._act() # 这个rsp是否需要publish_message? + rsp = await self._act() actions_taken += 1 return rsp # return output from the last action diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 1cb347a19..85e7dc8a4 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -183,10 +183,20 @@ class FileRepository: """ current_time = datetime.now().strftime("%Y%m%d%H%M%S") return current_time - # guid_suffix = str(uuid.uuid4())[:8] - # return f"{current_time}x{guid_suffix}" - async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: List[str] = None): + async def save_doc(self, doc: Document, dependencies: List[str] = None): + """Save content to a file and update its dependencies. + + :param doc: The Document instance to be saved. + :type doc: Document + :param dependencies: A list of dependencies for the saved file. + :type dependencies: List[str], optional + """ + + await self.save(filename=doc.filename, content=doc.content, dependencies=dependencies) + logger.debug(f"File Saved: {str(doc.filename)}") + + async def save_pdf(self, doc: Document, with_suffix: str = ".md", dependencies: List[str] = None): """Save a Document instance as a PDF file. This method converts the content of the Document instance to Markdown, diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index e9855df05..4feed89d5 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -199,10 +199,17 @@ class GitRepository: if new_path.exists(): logger.info(f"Delete directory {str(new_path)}") shutil.rmtree(new_path) + if new_path.exists(): # Recheck for windows os + logger.warning(f"Failed to delete directory {str(new_path)}") + return try: shutil.move(src=str(self.workdir), dst=str(new_path)) except Exception as e: logger.warning(f"Move {str(self.workdir)} to {str(new_path)} error: {e}") + finally: + if not new_path.exists(): # Recheck for windows os + logger.warning(f"Failed to move {str(self.workdir)} to {str(new_path)}") + return logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) self._gitignore_rules = parse_gitignore(full_path=str(new_path / ".gitignore")) diff --git a/metagpt/utils/project_repo.py b/metagpt/utils/project_repo.py index deedd6c03..71cb9d55d 100644 --- a/metagpt/utils/project_repo.py +++ b/metagpt/utils/project_repo.py @@ -17,9 +17,11 @@ from metagpt.const import ( CODE_SUMMARIES_PDF_FILE_REPO, COMPETITIVE_ANALYSIS_FILE_REPO, DATA_API_DESIGN_FILE_REPO, + DOCS_FILE_REPO, GRAPH_REPO_FILE_REPO, PRD_PDF_FILE_REPO, PRDS_FILE_REPO, + RESOURCES_FILE_REPO, SD_OUTPUT_FILE_REPO, SEQ_FLOW_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, @@ -33,7 +35,7 @@ from metagpt.utils.file_repository import FileRepository from metagpt.utils.git_repository import GitRepository -class DocFileRepositories: +class DocFileRepositories(FileRepository): prd: FileRepository system_design: FileRepository task: FileRepository @@ -42,6 +44,8 @@ class DocFileRepositories: class_view: FileRepository def __init__(self, git_repo): + super().__init__(git_repo=git_repo, relative_path=DOCS_FILE_REPO) + self.prd = git_repo.new_file_repository(relative_path=PRDS_FILE_REPO) self.system_design = git_repo.new_file_repository(relative_path=SYSTEM_DESIGN_FILE_REPO) self.task = git_repo.new_file_repository(relative_path=TASK_FILE_REPO) @@ -50,7 +54,7 @@ class DocFileRepositories: self.class_view = git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO) -class ResourceFileRepositories: +class ResourceFileRepositories(FileRepository): competitive_analysis: FileRepository data_api_design: FileRepository seq_flow: FileRepository @@ -61,6 +65,8 @@ class ResourceFileRepositories: sd_output: FileRepository def __init__(self, git_repo): + super().__init__(git_repo=git_repo, relative_path=RESOURCES_FILE_REPO) + self.competitive_analysis = git_repo.new_file_repository(relative_path=COMPETITIVE_ANALYSIS_FILE_REPO) self.data_api_design = git_repo.new_file_repository(relative_path=DATA_API_DESIGN_FILE_REPO) self.seq_flow = git_repo.new_file_repository(relative_path=SEQ_FLOW_FILE_REPO) @@ -72,16 +78,40 @@ class ResourceFileRepositories: class ProjectRepo(FileRepository): - def __init__(self, root: str | Path): - git_repo = GitRepository(local_path=Path(root)) - super().__init__(git_repo=git_repo, relative_path=Path(".")) + def __init__(self, root: str | Path = None, git_repo: GitRepository = None): + if not root and not git_repo: + raise ValueError("Invalid root and git_repo") + git_repo_ = git_repo or GitRepository(local_path=Path(root)) + super().__init__(git_repo=git_repo_, relative_path=Path(".")) - self._git_repo = git_repo + self._git_repo = git_repo_ self.docs = DocFileRepositories(self._git_repo) self.resources = ResourceFileRepositories(self._git_repo) self.tests = self._git_repo.new_file_repository(relative_path=TEST_CODES_FILE_REPO) self.test_outputs = self._git_repo.new_file_repository(relative_path=TEST_OUTPUTS_FILE_REPO) + self._srcs_path = None @property - def git_repo(self): + def git_repo(self) -> GitRepository: return self._git_repo + + @property + def workdir(self) -> Path: + return Path(self.git_repo.workdir) + + @property + def srcs(self) -> FileRepository: + if not self._srcs_path: + raise ValueError("Call with_srcs first.") + return self._git_repo.new_file_repository(self._srcs_path) + + def with_src_path(self, path: str | Path) -> ProjectRepo: + try: + self._srcs_path = Path(path).relative_to(self.workdir) + except ValueError: + self._srcs_path = Path(path) + return self + + @property + def src_relative_path(self) -> Path | None: + return self._srcs_path