diff --git a/metagpt/actions/fix_bug.py b/metagpt/actions/fix_bug.py new file mode 100644 index 000000000..6bd550d3d --- /dev/null +++ b/metagpt/actions/fix_bug.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023-12-12 +@Author : mashenquan +@File : fix_bug.py +""" +from metagpt.actions import Action + + +class FixBug(Action): + """Fix bug action without any implementation details""" + + async def run(self, *args, **kwargs): + raise NotImplementedError diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index b20539e78..1dda6466f 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -20,7 +20,8 @@ from tenacity import retry, stop_after_attempt, wait_random_exponential from metagpt.actions.action import Action from metagpt.config import CONFIG -from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO +from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO, BUGFIX_FILENAME, \ + DOCS_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, Document, RunCodeResult from metagpt.utils.common import CodeParser @@ -55,6 +56,12 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc {summary_log} ``` ----- +# Bug Feedback logs +```text +{feedback} +``` +----- + ## Code: {filename} Write code with triple quoto, based on the following list and context. 1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. @@ -89,6 +96,7 @@ class WriteCode(Action): return code async def run(self, *args, **kwargs) -> CodingContext: + bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO) coding_context = CodingContext.loads(self.context.content) test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO @@ -108,6 +116,7 @@ class WriteCode(Action): tasks=coding_context.task_doc.content if coding_context.task_doc else "", code=code_context, logs=logs, + feedback=bug_feedback.content if bug_feedback else "", filename=self.context.filename, summary_log=summary_doc.content if summary_doc else "", ) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 530a22def..aad2422ef 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -17,6 +17,7 @@ from pathlib import Path from typing import List from metagpt.actions import Action, ActionOutput +from metagpt.actions.fix_bug import FixBug from metagpt.actions.search_and_summarize import SearchAndSummarize from metagpt.config import CONFIG from metagpt.const import ( @@ -24,10 +25,10 @@ from metagpt.const import ( DOCS_FILE_REPO, PRD_PDF_FILE_REPO, PRDS_FILE_REPO, - REQUIREMENT_FILENAME, + REQUIREMENT_FILENAME, BUGFIX_FILENAME, ) from metagpt.logs import logger -from metagpt.schema import Document, Documents +from metagpt.schema import Document, Documents, Message, BugFixContext from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template @@ -227,7 +228,6 @@ There are no unclear points. }, } - OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), @@ -305,15 +305,44 @@ output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old P and only output the json inside this tag, nothing else """ +IS_BUGFIX_PROMPT = """ +{content} + +___ +You are a professional product manager; You need to determine whether the above content describes a requirement or provides feedback about a bug. +Respond with `YES` if it is a feedback about a bug, `NO` if it is not, and provide the reasons. Return the response in JSON format like below: + +```json +{{ + "is_bugfix": ..., # `YES` or `NO` + "reason": ..., # reason string +}} +``` +""" + class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: + async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput | Message: # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are # related to the PRD. If they are related, rewrite the PRD. - requirement_doc = await FileRepository.get_file(filename=REQUIREMENT_FILENAME, relative_path=DOCS_FILE_REPO) + docs_file_repo = CONFIG.git_repo.new_file_repository(relative_path=DOCS_FILE_REPO) + requirement_doc = await docs_file_repo.get(filename=REQUIREMENT_FILENAME) + if await self._is_bugfix(requirement_doc.content): + await docs_file_repo.save(filename=BUGFIX_FILENAME, content=requirement_doc.content) + await docs_file_repo.save(filename=REQUIREMENT_FILENAME, content="") + bug_fix = BugFixContext(filename=BUGFIX_FILENAME) + return Message(content=bug_fix.json(), instruct_content=bug_fix, + role=self.profile, + cause_by=FixBug, + sent_from=self, + send_to="Alex", # the name of Engineer + ) + else: + await docs_file_repo.delete(filename=BUGFIX_FILENAME) + prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) prd_docs = await prds_file_repo.get_all() change_files = Documents() @@ -405,7 +434,7 @@ class WritePRD(Action): if not quadrant_chart: return pathname = ( - CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") + CONFIG.git_repo.workdir / Path(COMPETITIVE_ANALYSIS_FILE_REPO) / Path(prd_doc.filename).with_suffix("") ) if not pathname.parent.exists(): pathname.parent.mkdir(parents=True, exist_ok=True) @@ -430,3 +459,11 @@ class WritePRD(Action): ws_name = CodeParser.parse_str(block="Project Name", text=prd) CONFIG.project_name = ws_name CONFIG.git_repo.rename_root(CONFIG.project_name) + + async def _is_bugfix(self, content): + prompt = IS_BUGFIX_PROMPT.format(content=content) + res = await self._aask(prompt=prompt) + logger.info(f"IS_BUGFIX:{res}") + if "YES" in res: + return True + return False diff --git a/metagpt/const.py b/metagpt/const.py index bd735a5e1..f6f64a27d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -74,6 +74,7 @@ MESSAGE_ROUTE_TO_ALL = "" MESSAGE_ROUTE_TO_NONE = "" REQUIREMENT_FILENAME = "requirement.txt" +BUGFIX_FILENAME = "bugfix.txt" PACKAGE_REQUIREMENTS_FILENAME = "requirements.txt" DOCS_FILE_REPO = "docs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 9f8eb6482..cedd2101f 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -24,6 +24,7 @@ from pathlib import Path from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks +from metagpt.actions.fix_bug import FixBug from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( @@ -78,7 +79,7 @@ class Engineer(Role): """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -191,14 +192,14 @@ class Engineer(Role): async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode, FixBug]) summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) if not self._rc.news: return None msg = self._rc.news[0] if msg.cause_by in write_code_filters: logger.info(f"TODO WriteCode:{msg.json()}") - await self._new_code_actions() + await self._new_code_actions(bug_fix=msg.cause_by == any_to_str(FixBug)) return self._rc.todo if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): logger.info(f"TODO SummarizeCode:{msg.json()}") @@ -232,10 +233,10 @@ class Engineer(Role): coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json()) return coding_doc - async def _new_code_actions(self): + async def _new_code_actions(self, bug_fix=False): # Prepare file repos src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + changed_src_files = src_file_repo.all_files if bug_fix else src_file_repo.changed_files task_file_repo = CONFIG.git_repo.new_file_repository(TASK_FILE_REPO) changed_task_files = task_file_repo.changed_files design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 2651be7eb..52ac3cf28 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -286,6 +286,8 @@ class Role: cause_by=self._rc.todo, sent_from=self, ) + elif isinstance(response, Message): + msg = response else: msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) diff --git a/metagpt/schema.py b/metagpt/schema.py index a8c1b7726..25281e399 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -97,14 +97,14 @@ class Message(BaseModel): send_to: Set = Field(default_factory={MESSAGE_ROUTE_TO_ALL}) def __init__( - self, - content, - instruct_content=None, - role="user", - cause_by="", - sent_from="", - send_to=MESSAGE_ROUTE_TO_ALL, - **kwargs, + self, + content, + instruct_content=None, + role="user", + cause_by="", + sent_from="", + send_to=MESSAGE_ROUTE_TO_ALL, + **kwargs, ): """ Parameters not listed below will be stored as meta info, including custom parameters. @@ -341,3 +341,7 @@ class CodeSummarizeContext(BaseModel): def __hash__(self): return hash((self.design_filename, self.task_filename)) + + +class BugFixContext(BaseModel): + filename: str = "" diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 5aec4509c..d372fd22e 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -8,15 +8,13 @@ """ from __future__ import annotations -import os +from gitignore_parser import parse_gitignore, rule_from_pattern, handle_negation import shutil from enum import Enum from pathlib import Path from typing import Dict, List - from git.repo import Repo from git.repo.fun import is_git_dir - from metagpt.const import DEFAULT_WORKSPACE_ROOT from metagpt.logs import logger from metagpt.utils.dependency_file import DependencyFile @@ -51,6 +49,7 @@ class GitRepository: """ self._repository = None self._dependency = None + self._gitignore_rules = None if local_path: self.open(local_path=local_path, auto_init=auto_init) @@ -63,6 +62,7 @@ class GitRepository: local_path = Path(local_path) if self.is_git_dir(local_path): self._repository = Repo(local_path) + self._gitignore_rules = parse_gitignore(full_path=str(local_path / ".gitignore")) return if not auto_init: return @@ -82,6 +82,7 @@ class GitRepository: writer.write("\n".join(ignores)) self._repository.index.add([".gitignore"]) self._repository.index.commit("Add .gitignore") + self._gitignore_rules = parse_gitignore(full_path=gitignore_filename) def add_change(self, files: Dict): """Add or remove files from the staging area based on the provided changes. @@ -204,8 +205,9 @@ class GitRepository: logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) - def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None) -> List: - """Retrieve a list of files in the specified relative path. + def get_files(self, relative_path: Path | str, root_relative_path: Path | str = None, filter_ignored=True) -> List: + """ + Retrieve a list of files in the specified relative path. The method returns a list of file paths relative to the current FileRepository. @@ -213,6 +215,8 @@ class GitRepository: :type relative_path: Path or str :param root_relative_path: The root relative path within the repository. :type root_relative_path: Path or str + :param filter_ignored: Flag to indicate whether to filter files based on .gitignore rules. + :type filter_ignored: bool :return: A list of file paths in the specified directory. :rtype: List[str] """ @@ -231,10 +235,35 @@ class GitRepository: rpath = file_path.relative_to(root_relative_path) files.append(str(rpath)) else: - subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path) + subfolder_files = self.get_files(relative_path=file_path, root_relative_path=root_relative_path, + filter_ignored=False) files.extend(subfolder_files) except Exception as e: logger.error(f"Error: {e}") + if not filter_ignored: + return files + filtered_files = self.filter_gitignore(filenames=files, root_relative_path=root_relative_path) + return filtered_files + + def filter_gitignore(self, filenames: List[str], root_relative_path: Path | str = None) -> List[str]: + """ + Filter a list of filenames based on .gitignore rules. + + :param filenames: A list of filenames to be filtered. + :type filenames: List[str] + :param root_relative_path: The root relative path within the repository. + :type root_relative_path: Path or str + :return: A list of filenames that pass the .gitignore filtering. + :rtype: List[str] + """ + if root_relative_path is None: + root_relative_path = self.workdir + files = [] + for filename in filenames: + pathname = root_relative_path / filename + if self._gitignore_rules(str(pathname)): + continue + files.append(filename) return files @@ -244,6 +273,7 @@ if __name__ == "__main__": repo = GitRepository() repo.open(path, auto_init=True) + repo.filter_gitignore(filenames=["snake_game/snake_game/__pycache__", "snake_game/snake_game/game.py"]) changes = repo.changed_files print(changes) diff --git a/requirements.txt b/requirements.txt index 99f738448..515a4d88b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -48,4 +48,4 @@ websocket-client==0.58.0 aiofiles==23.2.1 gitpython==3.1.40 zhipuai==1.0.7 - +gitignore-parser==0.1.9 diff --git a/tests/metagpt/utils/test_git_repository.py b/tests/metagpt/utils/test_git_repository.py index 23bebba7f..d800e9594 100644 --- a/tests/metagpt/utils/test_git_repository.py +++ b/tests/metagpt/utils/test_git_repository.py @@ -73,6 +73,13 @@ async def test_git1(): repo1 = GitRepository(local_path=local_path, auto_init=False) assert repo1.changed_files + file_repo = repo1.new_file_repository("__pycache__") + await file_repo.save("a.pyc", content="") + all_files = repo1.get_files(relative_path=".", filter_ignored=False) + assert "__pycache__/a.pyc" in all_files + all_files = repo1.get_files(relative_path=".", filter_ignored=True) + assert "__pycache__/a.pyc" not in all_files + repo1.delete_repository() assert not local_path.exists()