From 2bf8ef8c6ad18808447b827b6699e89650d7170c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 22 Nov 2023 17:08:00 +0800 Subject: [PATCH] feat: RFC 135 --- metagpt/actions/design_api.py | 47 +++++++- metagpt/actions/prepare_documents.py | 38 +++++-- metagpt/actions/write_prd.py | 38 ++++++- metagpt/config.py | 3 +- metagpt/const.py | 5 + metagpt/environment.py | 9 +- metagpt/roles/product_manager.py | 7 +- metagpt/schema.py | 40 ++++++- metagpt/utils/dependency_file.py | 83 ++++++++++++++ metagpt/utils/file_repository.py | 116 ++++++++++++-------- metagpt/utils/git_repository.py | 15 ++- requirements.txt | 2 +- startup.py | 6 + tests/metagpt/utils/test_dependency_file.py | 64 +++++++++++ tests/metagpt/utils/test_file_repository.py | 10 +- tests/metagpt/utils/test_git_repository.py | 15 +++ 16 files changed, 416 insertions(+), 82 deletions(-) create mode 100644 metagpt/utils/dependency_file.py create mode 100644 tests/metagpt/utils/test_dependency_file.py diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 75df8b909..65d53364b 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -11,8 +11,9 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG -from metagpt.const import WORKSPACE_ROOT +from metagpt.const import PRDS_FILE_REPO, SYS_DESIGN_FILE_REPO, WORKSPACE_ROOT from metagpt.logs import logger +from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template from metagpt.utils.json_to_markdown import json_to_markdown @@ -202,7 +203,44 @@ class WriteDesign(Action): await self._save_prd(docs_path, resources_path, context) await self._save_system_design(docs_path, resources_path, system_design) - async def run(self, context, format=CONFIG.prompt_format): + async def run(self, with_messages, format=CONFIG.prompt_format): + # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + changed_prds = prds_file_repo.changed_files + # 通过git diff来识别docs/system_designs下那些设计文档发生了变动； + system_design_file_repo = CONFIG.git_repo.new_file_repository(SYS_DESIGN_FILE_REPO) + changed_system_designs = system_design_file_repo.changed_files + + # 对于那些发生变动的PRD和设计文档，重新生成设计内容； + changed_files = Documents() + for filename in changed_prds.keys(): + prd = await prds_file_repo.get(filename) + old_system_design_doc = await system_design_file_repo.get(filename) + if not old_system_design_doc: + system_design = await self._run(context=prd.content) + doc = Document( + root_path=SYS_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json() + ) + else: + doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc) + await system_design_file_repo.save( + filename=filename, content=doc.content, dependencies={prd.root_relative_path} + ) + changed_files.docs[filename] = doc + + for filename in changed_system_designs.keys(): + if filename in changed_files.docs: + continue + prd_doc = await prds_file_repo.get(filename=filename) + old_system_design_doc = await system_design_file_repo.get(filename) + new_system_design_doc = await self._merge(prd_doc, old_system_design_doc) + await system_design_file_repo.save(filename=filename, content=new_system_design_doc.content) + changed_files.docs[filename] = new_system_design_doc + + # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 + return ActionOutput(content=changed_files.json(), instruct_content=changed_files) + + async def _run(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) prompt = prompt_template.format(context=context, format_example=format_example) # system_design = await self._aask(prompt) @@ -213,5 +251,8 @@ class WriteDesign(Action): "Python package name", system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), ) - await self._save(context, system_design) + # await self._save(context, system_design) return system_design + + async def _merge(self, prd_doc, system_design_doc): + return system_design_doc diff --git a/metagpt/actions/prepare_documents.py b/metagpt/actions/prepare_documents.py index b0185996b..c9b60ff27 100644 --- a/metagpt/actions/prepare_documents.py +++ b/metagpt/actions/prepare_documents.py @@ -7,19 +7,37 @@ @Desc: PrepareDocuments Action: initialize project folder and add new requirements to docs/requirements.txt. RFC 135 2.2.3.5.1. """ -from metagpt.actions import Action + +from pathlib import Path + +from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, REQUIREMENT_FILENAME, WORKSPACE_ROOT +from metagpt.schema import Document +from metagpt.utils.file_repository import FileRepository +from metagpt.utils.git_repository import GitRepository class PrepareDocuments(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_message, **kwargs): - parent = self.context.get("parent") - if not parent: - raise ValueError("Invalid owner") - env = parent.get_env() - if env.git_repository: - return - env.git_repository = GitRepository() - env.git_repository.open(WORKS) + async def run(self, with_messages, **kwargs): + if CONFIG.git_repo: + docs_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + doc = await docs_repo.get(REQUIREMENT_FILENAME) + return ActionOutput(content=doc.json(exclue="content"), instruct_content=doc) + + # Create and initialize the workspace folder, initialize the Git environment. + CONFIG.git_repo = GitRepository() + workdir = Path(CONFIG.WORKDIR) if CONFIG.WORKDIR else WORKSPACE_ROOT / FileRepository.new_file_name() + CONFIG.git_repo.open(local_path=workdir, auto_init=True) + + # Write the newly added requirements from the main parameter idea to `docs/requirement.txt`. + docs_file_repository = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + doc = Document(root_path=DOCS_FILE_REPO, filename=REQUIREMENT_FILENAME, content=with_messages[0].content) + await docs_file_repository.save(REQUIREMENT_FILENAME, content=doc.content) + + # Send a Message notification to the WritePRD action, instructing it to process requirements using + # `docs/requirement.txt` and `docs/prds/`. + return ActionOutput(content=doc.content, instruct_content=doc) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index bd04ca79e..a16d1ec06 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -10,7 +10,10 @@ from typing import List from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG +from metagpt.const import DOCS_FILE_REPO, PRDS_FILE_REPO, REQUIREMENT_FILENAME from metagpt.logs import logger +from metagpt.schema import Document, Documents +from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template templates = { @@ -222,7 +225,34 @@ class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + # 判断哪些需求文档需要重写：调LLM判断新增需求与prd是否相关，若相关就rewrite prd + docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) + requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) + prd_docs = await prds_file_repo.get_all() + change_files = Documents() + for prd_doc in prd_docs: + if await self._is_relative_to(requirement_doc, prd_doc): + prd_doc = await self._merge(requirement_doc, prd_doc) + await prds_file_repo.save(filename=prd_doc.filename, content=prd_doc.content) + change_files.docs[prd_doc.filename] = prd_doc + # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd + if not change_files.docs: + prd = await self._run_new_requirement( + requirements=[requirement_doc.content], format=format, *args, **kwargs + ) + doc = Document( + root_path=PRDS_FILE_REPO, + filename=FileRepository.new_file_name() + ".json", + content=prd.instruct_content.json(), + ) + await prds_file_repo.save(filename=doc.filename, content=doc.content) + change_files.docs[doc.filename] = doc + # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 + return ActionOutput(content=change_files.json(), instruct_content=change_files) + + async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: sas = SearchAndSummarize() # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" @@ -239,3 +269,9 @@ class WritePRD(Action): # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd + + async def _is_relative_to(self, doc1, doc2) -> bool: + return False + + async def _merge(self, doc1, doc2) -> Document: + pass diff --git a/metagpt/config.py b/metagpt/config.py index 27455d38d..51eed4fb8 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -46,7 +46,7 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -93,6 +93,7 @@ class Config(metaclass=Singleton): self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "") self.prompt_format = self._get("PROMPT_FORMAT", "markdown") + self.git_repo = None def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority""" diff --git a/metagpt/const.py b/metagpt/const.py index fa0ccc536..fc1c47b5b 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -49,3 +49,8 @@ MESSAGE_ROUTE_TO = "send_to" MESSAGE_ROUTE_CAUSE_BY = "cause_by" MESSAGE_META_ROLE = "role" MESSAGE_ROUTE_TO_ALL = "" + +REQUIREMENT_FILENAME = "requirement.txt" +DOCS_FILE_REPO = "docs" +PRDS_FILE_REPO = "docs/prds" +SYS_DESIGN_FILE_REPO = "docs/system_design" diff --git a/metagpt/environment.py b/metagpt/environment.py index df93a818b..b3c296dac 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -12,7 +12,7 @@ functionality is to be consolidated into the `Environment` class. """ import asyncio -from typing import Iterable, Optional, Set +from typing import Iterable, Set from pydantic import BaseModel, Field @@ -20,7 +20,6 @@ from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import is_subscribed -from metagpt.utils.git_repository import GitRepository class Environment(BaseModel): @@ -32,7 +31,6 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) consumers: dict[Role, Set] = Field(default_factory=dict) history: str = Field(default="") # For debug - git_repository: Optional[GitRepository] = None class Config: arbitrary_types_allowed = True @@ -113,8 +111,3 @@ class Environment(BaseModel): def set_subscription(self, obj, tags): """Set the labels for message to be consumed by the object""" self.consumers[obj] = tags - - def dict(self, *args, **kwargs): - """Generate a dictionary representation of the model, optionally specifying which fields to include or - exclude.""" - return super(Environment, self).dict(exclude={"git_repository"}) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index c10aba6d1..81577ec2c 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -7,6 +7,7 @@ """ from metagpt.actions import BossRequirement, WritePRD from metagpt.actions.prepare_documents import PrepareDocuments +from metagpt.config import CONFIG from metagpt.roles import Role @@ -38,12 +39,12 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([PrepareDocuments(context={"parent": self}), WritePRD]) - self._watch([BossRequirement]) + self._init_actions([PrepareDocuments, WritePRD]) + self._watch([BossRequirement, PrepareDocuments]) async def _think(self) -> None: """Decide what to do""" - if self._rc.env.git_repository: + if CONFIG.git_repo: self._set_state(1) else: self._set_state(0) diff --git a/metagpt/schema.py b/metagpt/schema.py index 82a0117ef..674091e4c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -6,14 +6,16 @@ @File : schema.py @Modified By: mashenquan, 2023-10-31. According to Chapter 2.2.1 of RFC 116: Replanned the distribution of responsibilities and functional positioning of `Message` class attributes. +@Modified By: mashenquan, 2023/11/22. Add `Document` and `Documents` for `FileRepository` in Section 2.2.3.4 of RFC 135. """ from __future__ import annotations import asyncio import json +import os.path from asyncio import Queue, QueueEmpty, wait_for from json import JSONDecodeError -from typing import List, Set, TypedDict +from typing import Dict, List, Optional, Set, TypedDict from pydantic import BaseModel, Field @@ -32,6 +34,42 @@ class RawMessage(TypedDict): role: str +class Document(BaseModel): + """ + Represents a document. + """ + + root_path: str + filename: str + content: Optional[str] = None + + def get_meta(self) -> Document: + """Get metadata of the document. + + :return: A new Document instance with the same root path and filename. + """ + + return Document(root_path=self.root_path, filename=self.filename) + + @property + def root_relative_path(self): + """Get relative path from root of git repository. + + :return: relative path from root of git repository. + """ + return os.path.join(self.root_path, self.filename) + + +class Documents(BaseModel): + """A class representing a collection of documents. + + Attributes: + docs (Dict[str, Document]): A dictionary mapping document names to Document instances. + """ + + docs: Dict[str, Document] = Field(default_factory=dict) + + class Message(BaseModel): """list[: ]""" diff --git a/metagpt/utils/dependency_file.py b/metagpt/utils/dependency_file.py new file mode 100644 index 000000000..429027c7a --- /dev/null +++ b/metagpt/utils/dependency_file.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : dependency_file.py +@Desc: Implementation of the dependency file described in Section 2.2.3.2 of RFC 135. +""" +from __future__ import annotations + +import json +from pathlib import Path +from typing import Set + +import aiofiles + +from metagpt.logs import logger + + +class DependencyFile: + def __init__(self, workdir: Path | str): + self._dependencies = {} + self._filename = Path(workdir) / ".dependencies.json" + + async def load(self): + if not self._filename.exists(): + return + try: + async with aiofiles.open(str(self._filename), mode="r") as reader: + data = await reader.read() + self._dependencies = json.loads(data) + except Exception as e: + logger.error(f"Failed to load {str(self._filename)}, error:{e}") + + async def save(self): + try: + data = json.dumps(self._dependencies) + async with aiofiles.open(str(self._filename), mode="w") as writer: + await writer.write(data) + except Exception as e: + logger.error(f"Failed to save {str(self._filename)}, error:{e}") + + async def update(self, filename: Path | str, dependencies: Set[Path | str], persist=True): + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + + if dependencies: + relative_paths = [] + for i in dependencies: + try: + relative_paths.append(str(Path(i).relative_to(root))) + except ValueError: + relative_paths.append(str(i)) + self._dependencies[str(key)] = relative_paths + elif str(key) in self._dependencies: + del self._dependencies[str(key)] + + if persist: + await self.save() + + async def get(self, filename: Path | str, persist=False): + if persist: + await self.load() + + root = self._filename.parent + try: + key = Path(filename).relative_to(root) + except ValueError: + key = filename + return set(self._dependencies.get(str(key), {})) + + def delete_file(self): + self._filename.unlink(missing_ok=True) + + @property + def exists(self): + return self._filename.exists() diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index f4c36b5b7..7f07e4427 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -8,16 +8,29 @@ """ from __future__ import annotations -import json +import os +import uuid +from datetime import datetime from pathlib import Path -from typing import Dict, List +from typing import Dict, List, Set import aiofiles from metagpt.logs import logger +from metagpt.schema import Document class FileRepository: + """A class representing a FileRepository associated with a Git repository. + + :param git_repo: The associated GitRepository instance. + :param relative_path: The relative path within the Git repository. + + Attributes: + _relative_path (Path): The relative path within the Git repository. + _git_repo (GitRepository): The associated GitRepository instance. + """ + def __init__(self, git_repo, relative_path: Path = Path(".")): """Initialize a FileRepository instance. @@ -26,16 +39,9 @@ class FileRepository: """ self._relative_path = relative_path self._git_repo = git_repo - self._dependencies: Dict[str, List[str]] = {} # Initializing self.workdir.mkdir(parents=True, exist_ok=True) - if self.dependency_path_name.exists(): - try: - with open(str(self.dependency_path_name), mode="r") as reader: - self._dependencies = json.load(reader) - except Exception as e: - logger.error(f"Failed to load {str(self.dependency_path_name)}, error:{e}") async def save(self, filename: Path | str, content, dependencies: List[str] = None): """Save content to a file and update its dependencies. @@ -44,59 +50,68 @@ class FileRepository: :param content: The content to be saved. :param dependencies: List of dependency filenames or paths. """ - path_name = self.workdir / filename - path_name.parent.mkdir(parents=True, exist_ok=True) - async with aiofiles.open(str(path_name), mode="w") as writer: + pathname = self.workdir / filename + pathname.parent.mkdir(parents=True, exist_ok=True) + async with aiofiles.open(str(pathname), mode="w") as writer: await writer.write(content) + logger.info(f"save to: {str(pathname)}") + if dependencies is not None: - await self.update_dependency(filename, dependencies) + dependency_file = await self._git_repo.get_dependency() + await dependency_file.update(pathname, set(dependencies)) + logger.info(f"update dependency: {str(pathname)}:{dependencies}") - async def get(self, filename: Path | str): - """Read the content of a file. - - :param filename: The filename or path within the repository. - :return: The content of the file. - """ - path_name = self.workdir / filename - async with aiofiles.open(str(path_name), mode="r") as reader: - return await reader.read() - - def get_dependency(self, filename: Path | str) -> List: + async def get_dependency(self, filename: Path | str) -> Set[str]: """Get the dependencies of a file. :param filename: The filename or path within the repository. - :return: List of dependency filenames or paths. + :return: Set of dependency filenames or paths. """ - key = str(filename) - return self._dependencies.get(key, []) + pathname = self.workdir / filename + dependency_file = await self._git_repo.get_dependency() + return await dependency_file.get(pathname) - def get_changed_dependency(self, filename: Path | str) -> List: + async def get_changed_dependency(self, filename: Path | str) -> Set[str]: """Get the dependencies of a file that have changed. :param filename: The filename or path within the repository. :return: List of changed dependency filenames or paths. """ - dependencies = self.get_dependency(filename=filename) + dependencies = await self.get_dependency(filename=filename) changed_files = self.changed_files - changed_dependent_files = [] + changed_dependent_files = set() for df in dependencies: if df in changed_files.keys(): - changed_dependent_files.append(df) + changed_dependent_files.add(df) return changed_dependent_files - async def update_dependency(self, filename, dependencies: List[str]): - """Update the dependencies of a file. + async def get(self, filename: Path | str) -> Document | None: + """Read the content of a file. :param filename: The filename or path within the repository. - :param dependencies: List of dependency filenames or paths. + :return: The content of the file. """ - self._dependencies[str(filename)] = dependencies + doc = Document(root_path=str(self.root_path), filename=str(filename)) + path_name = self.workdir / filename + if not path_name.exists(): + return None + async with aiofiles.open(str(path_name), mode="r") as reader: + doc.content = await reader.read() + return doc - async def save_dependency(self): - """Save the dependencies to a file.""" - data = json.dumps(self._dependencies) - with aiofiles.open(str(self.dependency_path_name), mode="w") as writer: - await writer.write(data) + async def get_all(self) -> List[Document]: + """Get the content of all files in the repository. + + :return: List of Document instances representing files. + """ + docs = [] + for root, dirs, files in os.walk(str(self.workdir)): + for file in files: + file_path = Path(root) / file + relative_path = file_path.relative_to(self.workdir) + doc = await self.get(relative_path) + docs.append(doc) + return docs @property def workdir(self): @@ -107,14 +122,9 @@ class FileRepository: return self._git_repo.workdir / self._relative_path @property - def dependency_path_name(self): - """Return the absolute path to the dependency file. - - :return: The absolute path to the dependency file. - """ - filename = ".dependencies.json" - path_name = self.workdir / filename - return path_name + def root_path(self): + """Return the relative path from git repository root""" + return self._relative_path @property def changed_files(self) -> Dict[str, str]: @@ -147,3 +157,13 @@ class FileRepository: continue children.append(str(f)) return children + + @staticmethod + def new_file_name(): + """Generate a new filename based on the current timestamp and a UUID suffix. + + :return: A new filename string. + """ + current_time = datetime.now().strftime("%Y%m%d%H%M%S") + guid_suffix = str(uuid.uuid4())[:8] + return f"{current_time}t{guid_suffix}" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 6ae6a7900..a81b5c4ea 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -17,6 +17,7 @@ from git.repo import Repo from git.repo.fun import is_git_dir from metagpt.const import WORKSPACE_ROOT +from metagpt.utils.dependency_file import DependencyFile from metagpt.utils.file_repository import FileRepository @@ -47,6 +48,7 @@ class GitRepository: :param auto_init: If True, automatically initializes a new Git repository if the provided path is not a Git repository. """ self._repository = None + self._dependency = None if local_path: self.open(local_path=local_path, auto_init=auto_init) @@ -113,7 +115,7 @@ class GitRepository: :param local_path: The local path to check. :return: True if the directory is a Git repository, False otherwise. """ - git_dir = local_path / ".git" + git_dir = Path(local_path) / ".git" if git_dir.exists() and is_git_dir(git_dir): return True return False @@ -151,7 +153,7 @@ class GitRepository: self.add_change(self.changed_files) self.commit(comments) - def new_file_repository(self, relative_path: Path | str) -> FileRepository: + def new_file_repository(self, relative_path: Path | str = ".") -> FileRepository: """Create a new instance of FileRepository associated with this Git repository. :param relative_path: The relative path to the file repository within the Git repository. @@ -159,6 +161,15 @@ class GitRepository: """ return FileRepository(git_repo=self, relative_path=Path(relative_path)) + async def get_dependency(self) -> DependencyFile: + """Get the dependency file associated with the Git repository. + + :return: An instance of DependencyFile. + """ + if not self._dependency: + self._dependency = DependencyFile(workdir=self.workdir) + return self._dependency + if __name__ == "__main__": path = WORKSPACE_ROOT / "git" diff --git a/requirements.txt b/requirements.txt index c3b909e77..73a03d537 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,4 +44,4 @@ ta==0.10.2 semantic-kernel==0.3.13.dev0 wrapt==1.15.0 websocket-client==0.58.0 - +aiofiles==23.2.1 diff --git a/startup.py b/startup.py index e2a903c9b..d5a6bb07b 100644 --- a/startup.py +++ b/startup.py @@ -4,6 +4,7 @@ import asyncio import fire +from metagpt.config import CONFIG from metagpt.roles import ( Architect, Engineer, @@ -54,6 +55,7 @@ def main( code_review: bool = True, run_tests: bool = False, implement: bool = True, + project_path: str = None, ): """ We are a software startup comprised of AI. By investing in us, @@ -63,8 +65,12 @@ def main( a certain dollar amount to this AI company. :param n_round: :param code_review: Whether to use code review. + :param run_tests: Whether run unit tests. + :param implement: Whether to write codes. + :param project_path: The path of the old version project to improve. :return: """ + CONFIG.WORKDIR = project_path asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) diff --git a/tests/metagpt/utils/test_dependency_file.py b/tests/metagpt/utils/test_dependency_file.py new file mode 100644 index 000000000..ae4d40ea5 --- /dev/null +++ b/tests/metagpt/utils/test_dependency_file.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/11/22 +@Author : mashenquan +@File : test_dependency_file.py +@Desc: Unit tests for dependency_file.py +""" +from __future__ import annotations + +from pathlib import Path +from typing import Optional, Set, Union + +import pytest +from pydantic import BaseModel + +from metagpt.utils.dependency_file import DependencyFile + + +@pytest.mark.asyncio +async def test_dependency_file(): + class Input(BaseModel): + x: Union[Path, str] + deps: Optional[Set[Union[Path, str]]] + key: Optional[Union[Path, str]] + want: Set[str] + + inputs = [ + Input(x="a/b.txt", deps={"c/e.txt", Path(__file__).parent / "d.txt"}, want={"c/e.txt", "d.txt"}), + Input( + x=Path(__file__).parent / "x/b.txt", + deps={"s/e.txt", Path(__file__).parent / "d.txt"}, + key="x/b.txt", + want={"s/e.txt", "d.txt"}, + ), + Input(x="f.txt", deps=None, want=set()), + Input(x="a/b.txt", deps=None, want=set()), + ] + + file = DependencyFile(workdir=Path(__file__).parent) + + for i in inputs: + await file.update(filename=i.x, dependencies=i.deps) + assert await file.get(filename=i.key or i.x) == i.want + + file2 = DependencyFile(workdir=Path(__file__).parent) + file2.delete_file() + assert not file.exists + await file2.update(filename="a/b.txt", dependencies={"c/e.txt", Path(__file__).parent / "d.txt"}, persist=False) + assert not file.exists + await file2.save() + assert file2.exists + + file1 = DependencyFile(workdir=Path(__file__).parent) + assert file1.exists + assert await file1.get("a/b.txt") == set() + await file1.load() + assert await file1.get("a/b.txt") == {"c/e.txt", "d.txt"} + file1.delete_file() + assert not file.exists + + +if __name__ == "__main__": + pytest.main([__file__, "-s"]) diff --git a/tests/metagpt/utils/test_file_repository.py b/tests/metagpt/utils/test_file_repository.py index ac36f2320..a830b58aa 100644 --- a/tests/metagpt/utils/test_file_repository.py +++ b/tests/metagpt/utils/test_file_repository.py @@ -34,11 +34,13 @@ async def test_file_repo(): assert file_repo.workdir.exists() await file_repo.save("a.txt", "AAA") await file_repo.save("b.txt", "BBB", ["a.txt"]) - assert "AAA" == await file_repo.get("a.txt") - assert "BBB" == await file_repo.get("b.txt") - assert ["a.txt"] == file_repo.get_dependency("b.txt") + doc = await file_repo.get("a.txt") + assert "AAA" == doc.content + doc = await file_repo.get("b.txt") + assert "BBB" == doc.content + assert {"a.txt"} == await file_repo.get_dependency("b.txt") assert {"a.txt": ChangeType.UNTRACTED, "b.txt": ChangeType.UNTRACTED} == file_repo.changed_files - assert ["a.txt"] == file_repo.get_changed_dependency("b.txt") + assert {"a.txt"} == await file_repo.get_changed_dependency("b.txt") await file_repo.save("d/e.txt", "EEE") assert ["d/e.txt"] == file_repo.get_change_dir_files("d") diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 0d1e3b791..23bebba7f 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -77,5 +77,20 @@ async def test_git1(): assert not local_path.exists() +@pytest.mark.asyncio +async def test_dependency_file(): + local_path = Path(__file__).parent / "git2" + repo, subdir = await mock_repo(local_path) + + dependancy_file = await repo.get_dependency() + assert not dependancy_file.exists + + await dependancy_file.update(filename="a/b.txt", dependencies={"c/d.txt", "e/f.txt"}) + assert dependancy_file.exists + + repo.delete_repository() + assert not dependancy_file.exists + + if __name__ == "__main__": pytest.main([__file__, "-s"])