From 102ae2ca672f1ff69504f2c1578c7a9080216d35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 8 Jan 2024 22:15:43 +0800 Subject: [PATCH] feat: Implementation of ProjectRepo --- metagpt/const.py | 14 ++-- metagpt/utils/file_repository.py | 66 ------------------ metagpt/utils/project_repo.py | 87 ++++++++++++++++++++++++ tests/metagpt/utils/test_project_repo.py | 58 ++++++++++++++++ 4 files changed, 152 insertions(+), 73 deletions(-) create mode 100644 metagpt/utils/project_repo.py create mode 100644 tests/metagpt/utils/test_project_repo.py diff --git a/metagpt/const.py b/metagpt/const.py index 811ff9516..581aff5d3 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -89,23 +89,23 @@ BUGFIX_FILENAME = "bugfix.txt" PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt" DOCS_FILE_REPO = "docs" -PRDS_FILE_REPO = "docs/prds" +PRDS_FILE_REPO = "docs/prd" SYSTEM_DESIGN_FILE_REPO = "docs/system_design" -TASK_FILE_REPO = "docs/tasks" +TASK_FILE_REPO = "docs/task" COMPETITIVE_ANALYSIS_FILE_REPO = "resources/competitive_analysis" DATA_API_DESIGN_FILE_REPO = "resources/data_api_design" SEQ_FLOW_FILE_REPO = "resources/seq_flow" SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" -TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" +TASK_PDF_FILE_REPO = "resources/api_spec_and_task" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" -CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" -CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" +CODE_SUMMARIES_FILE_REPO = "docs/code_summary" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summary" RESOURCES_FILE_REPO = "resources" -SD_OUTPUT_FILE_REPO = "resources/SD_Output" +SD_OUTPUT_FILE_REPO = "resources/sd_output" GRAPH_REPO_FILE_REPO = "docs/graph_repo" -CLASS_VIEW_FILE_REPO = "docs/class_views" +CLASS_VIEW_FILE_REPO = "docs/class_view" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 3b5f5c5ac..01b78cd77 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -202,68 +202,6 @@ class FileRepository: await self.save(filename=str(filename), content=json_to_markdown(m), dependencies=dependencies) logger.debug(f"File Saved: {str(filename)}") - async def get_file(self, filename: Path | str, relative_path: Path | str = ".") -> Document | None: - """Retrieve a specific file from the file repository. - - :param filename: The name or path of the file to retrieve. - :type filename: Path or str - :param relative_path: The relative path within the file repository. - :type relative_path: Path or str, optional - :return: The document representing the file, or None if not found. - :rtype: Document or None - """ - file_repo = self._git_repo.new_file_repository(relative_path=relative_path) - return await file_repo.get(filename=filename) - - async def get_all_files(self, relative_path: Path | str = ".") -> List[Document]: - """Retrieve all files from the file repository. - - :param relative_path: The relative path within the file repository. - :type relative_path: Path or str, optional - :return: A list of documents representing all files in the repository. - :rtype: List[Document] - """ - file_repo = self._git_repo.new_file_repository(relative_path=relative_path) - return await file_repo.get_all() - - async def save_file( - self, filename: Path | str, content, dependencies: List[str] = None, relative_path: Path | str = "." - ): - """Save a file to the file repository. - - :param filename: The name or path of the file to save. - :type filename: Path or str - :param content: The content of the file. - :param dependencies: A list of dependencies for the file. - :type dependencies: List[str], optional - :param relative_path: The relative path within the file repository. - :type relative_path: Path or str, optional - """ - file_repo = self._git_repo.new_file_repository(relative_path=relative_path) - return await file_repo.save(filename=filename, content=content, dependencies=dependencies) - - async def save_as( - self, doc: Document, with_suffix: str = None, dependencies: List[str] = None, relative_path: Path | str = "." - ): - """Save a Document instance with optional modifications. - - This static method creates a new FileRepository, saves the Document instance - with optional modifications (such as a suffix), and logs the saved file. - - :param doc: The Document instance to be saved. - :type doc: Document - :param with_suffix: An optional suffix to append to the saved file's name. - :type with_suffix: str, optional - :param dependencies: A list of dependencies for the saved file. - :type dependencies: List[str], optional - :param relative_path: The relative path within the file repository. - :type relative_path: Path or str, optional - :return: A boolean indicating whether the save operation was successful. - :rtype: bool - """ - file_repo = self._git_repo.new_file_repository(relative_path=relative_path) - return await file_repo.save_doc(doc=doc, with_suffix=with_suffix, dependencies=dependencies) - async def delete(self, filename: Path | str): """Delete a file from the file repository. @@ -280,7 +218,3 @@ class FileRepository: dependency_file = await self._git_repo.get_dependency() await dependency_file.update(filename=pathname, dependencies=None) logger.info(f"remove dependency key: {str(pathname)}") - - async def delete_file(self, filename: Path | str, relative_path: Path | str = "."): - file_repo = self._git_repo.new_file_repository(relative_path=relative_path) - await file_repo.delete(filename=filename) diff --git a/metagpt/utils/project_repo.py b/metagpt/utils/project_repo.py new file mode 100644 index 000000000..deedd6c03 --- /dev/null +++ b/metagpt/utils/project_repo.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/1/8 +@Author : mashenquan +@File : project_repo.py +@Desc : Wrapper for GitRepository and FileRepository of project. + Implementation of Chapter 4.6 of https://deepwisdom.feishu.cn/wiki/CUK4wImd7id9WlkQBNscIe9cnqh +""" +from __future__ import annotations + +from pathlib import Path + +from metagpt.const import ( + CLASS_VIEW_FILE_REPO, + CODE_SUMMARIES_FILE_REPO, + CODE_SUMMARIES_PDF_FILE_REPO, + COMPETITIVE_ANALYSIS_FILE_REPO, + DATA_API_DESIGN_FILE_REPO, + GRAPH_REPO_FILE_REPO, + PRD_PDF_FILE_REPO, + PRDS_FILE_REPO, + SD_OUTPUT_FILE_REPO, + SEQ_FLOW_FILE_REPO, + SYSTEM_DESIGN_FILE_REPO, + SYSTEM_DESIGN_PDF_FILE_REPO, + TASK_FILE_REPO, + TASK_PDF_FILE_REPO, + TEST_CODES_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) +from metagpt.utils.file_repository import FileRepository +from metagpt.utils.git_repository import GitRepository + + +class DocFileRepositories: + prd: FileRepository + system_design: FileRepository + task: FileRepository + code_summary: FileRepository + graph_repo: FileRepository + class_view: FileRepository + + def __init__(self, git_repo): + self.prd = git_repo.new_file_repository(relative_path=PRDS_FILE_REPO) + self.system_design = git_repo.new_file_repository(relative_path=SYSTEM_DESIGN_FILE_REPO) + self.task = git_repo.new_file_repository(relative_path=TASK_FILE_REPO) + self.code_summary = git_repo.new_file_repository(relative_path=CODE_SUMMARIES_FILE_REPO) + self.graph_repo = git_repo.new_file_repository(relative_path=GRAPH_REPO_FILE_REPO) + self.class_view = git_repo.new_file_repository(relative_path=CLASS_VIEW_FILE_REPO) + + +class ResourceFileRepositories: + competitive_analysis: FileRepository + data_api_design: FileRepository + seq_flow: FileRepository + system_design: FileRepository + prd: FileRepository + api_spec_and_task: FileRepository + code_summary: FileRepository + sd_output: FileRepository + + def __init__(self, git_repo): + self.competitive_analysis = git_repo.new_file_repository(relative_path=COMPETITIVE_ANALYSIS_FILE_REPO) + self.data_api_design = git_repo.new_file_repository(relative_path=DATA_API_DESIGN_FILE_REPO) + self.seq_flow = git_repo.new_file_repository(relative_path=SEQ_FLOW_FILE_REPO) + self.system_design = git_repo.new_file_repository(relative_path=SYSTEM_DESIGN_PDF_FILE_REPO) + self.prd = git_repo.new_file_repository(relative_path=PRD_PDF_FILE_REPO) + self.api_spec_and_task = git_repo.new_file_repository(relative_path=TASK_PDF_FILE_REPO) + self.code_summary = git_repo.new_file_repository(relative_path=CODE_SUMMARIES_PDF_FILE_REPO) + self.sd_output = git_repo.new_file_repository(relative_path=SD_OUTPUT_FILE_REPO) + + +class ProjectRepo(FileRepository): + def __init__(self, root: str | Path): + git_repo = GitRepository(local_path=Path(root)) + super().__init__(git_repo=git_repo, relative_path=Path(".")) + + self._git_repo = git_repo + self.docs = DocFileRepositories(self._git_repo) + self.resources = ResourceFileRepositories(self._git_repo) + self.tests = self._git_repo.new_file_repository(relative_path=TEST_CODES_FILE_REPO) + self.test_outputs = self._git_repo.new_file_repository(relative_path=TEST_OUTPUTS_FILE_REPO) + + @property + def git_repo(self): + return self._git_repo diff --git a/tests/metagpt/utils/test_project_repo.py b/tests/metagpt/utils/test_project_repo.py new file mode 100644 index 000000000..6f80fbc14 --- /dev/null +++ b/tests/metagpt/utils/test_project_repo.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2024/1/8 +@Author : mashenquan +""" +import uuid +from pathlib import Path + +import pytest + +from metagpt.const import ( + BUGFIX_FILENAME, + PACKAGE_REQUIREMENTS_FILENAME, + PRDS_FILE_REPO, + REQUIREMENT_FILENAME, +) +from metagpt.utils.project_repo import ProjectRepo + + +async def test_project_repo(): + root = Path(__file__).parent / f"../../../workspace/unittest/{uuid.uuid4().hex}" + root = root.resolve() + + pr = ProjectRepo(root=str(root)) + assert pr.git_repo.workdir == root + + await pr.save(filename=REQUIREMENT_FILENAME, content=REQUIREMENT_FILENAME) + doc = await pr.get(filename=REQUIREMENT_FILENAME) + assert doc.content == REQUIREMENT_FILENAME + await pr.save(filename=BUGFIX_FILENAME, content=BUGFIX_FILENAME) + doc = await pr.get(filename=BUGFIX_FILENAME) + assert doc.content == BUGFIX_FILENAME + await pr.save(filename=PACKAGE_REQUIREMENTS_FILENAME, content=PACKAGE_REQUIREMENTS_FILENAME) + doc = await pr.get(filename=PACKAGE_REQUIREMENTS_FILENAME) + assert doc.content == PACKAGE_REQUIREMENTS_FILENAME + await pr.docs.prd.save(filename="1.prd", content="1.prd", dependencies=[REQUIREMENT_FILENAME]) + doc = await pr.docs.prd.get(filename="1.prd") + assert doc.content == "1.prd" + await pr.resources.prd.save( + filename="1.prd", + content="1.prd", + dependencies=[REQUIREMENT_FILENAME, f"{PRDS_FILE_REPO}/1.prd"], + ) + doc = await pr.resources.prd.get(filename="1.prd") + assert doc.content == "1.prd" + dependencies = await pr.resources.prd.get_dependency(filename="1.prd") + assert len(dependencies) == 2 + + assert pr.changed_files + assert pr.docs.prd.changed_files + assert not pr.tests.changed_files + + pr.git_repo.delete_repository() + + +if __name__ == "__main__": + pytest.main([__file__, "-s"])