From dac4be4b3e04f656c4f073e2161bd2c79c8eb242 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Dec 2023 23:04:07 +0800 Subject: [PATCH] feat: +SummarizeCode, refactor project_name --- metagpt/actions/design_api.py | 55 +++-------- metagpt/actions/project_management.py | 10 +- metagpt/actions/summarize_code.py | 8 +- metagpt/actions/write_code.py | 7 +- metagpt/actions/write_prd.py | 48 +++++++--- metagpt/const.py | 3 + metagpt/roles/engineer.py | 133 ++++++++++++++++++-------- metagpt/roles/qa_engineer.py | 12 ++- metagpt/roles/role.py | 3 +- metagpt/schema.py | 9 +- metagpt/utils/file_repository.py | 11 +++ metagpt/utils/git_repository.py | 28 +++++- 12 files changed, 224 insertions(+), 103 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index c5787ba20..605b871a1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023/11/27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ import json from pathlib import Path @@ -43,7 +44,7 @@ Requirement: Fill in the following missing information based on the context, eac ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text. ## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -58,15 +59,15 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Implementation approach": "We will ...", - "project_name": "snake_game", + "project_name": "{project_name}", "File list": ["main.py"], "Data structures and interfaces": ' classDiagram - class Game{ + class Game{{ +int score - } + }} ... Game "1" -- "1" Food: has ', @@ -77,7 +78,7 @@ and only output the json inside this tag, nothing else G->>M: end game ', "Anything UNCLEAR": "The requirement is clear to me." -} +}} [/CONTENT] """, }, @@ -96,7 +97,7 @@ ATTENTION: Output carefully referenced "Format example" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text. ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -114,7 +115,7 @@ We will ... ## project_name ```python -"snake_game" +"{project_name}" ``` ## File list @@ -173,7 +174,7 @@ ATTENTION: Output carefully referenced "Old Design" in format. ## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. -## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores +## project_name: Constant text "{project_name}". ## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here @@ -229,50 +230,20 @@ class WriteDesign(Action): async def _new_system_design(self, context, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) + format_example = format_example.format(project_name=CONFIG.project_name) prompt = prompt_template.format(context=context, format_example=format_example) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) - self._rename_project_name(system_design=system_design) - await self._rename_workspace(system_design) return system_design async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): - prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content, + project_name=CONFIG.project_name) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python # package name" contain space, have to use setattr - self._rename_project_name(system_design=system_design) system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False) return system_design_doc - @staticmethod - def _rename_project_name(system_design): - # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" - # contain space, have to use setattr - if CONFIG.project_name: - setattr( - system_design.instruct_content, - "project_name", - CONFIG.project_name, - ) - return - setattr( - system_design.instruct_content, - "project_name", - system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'), - ) - - @staticmethod - async def _rename_workspace(system_design): - if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to - # Section 2.2.3.10 of RFC 135 - return - - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["project_name"] - else: - ws_name = CodeParser.parse_str(block="project_name", text=system_design) - CONFIG.git_repo.rename_root(ws_name) - async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document: prd = await prds_file_repo.get(filename) old_system_design_doc = await system_design_file_repo.get(filename) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 3d59daeed..95da0d65a 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -183,6 +183,10 @@ MERGE_PROMPT = """ ## Old Tasks {old_tasks} ----- + +## Format example +{format_example} +----- Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -201,7 +205,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Format example" format, and only output the json inside this tag, nothing else """ @@ -264,7 +268,9 @@ class WriteTasks(Action): return rsp async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + _, format_example = get_template(templates, format) + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) task_doc.content = rsp.instruct_content.json(ensure_ascii=False) return task_doc diff --git a/metagpt/actions/summarize_code.py b/metagpt/actions/summarize_code.py index 88a37536b..d9cb47021 100644 --- a/metagpt/actions/summarize_code.py +++ b/metagpt/actions/summarize_code.py @@ -3,7 +3,9 @@ """ @Author : alexanderwu @File : summarize_code.py +@Modified By: mashenquan, 2023/12/5. Archive the summarization content of issue discovery for use in WriteCode. """ +from pathlib import Path from tenacity import retry, stop_after_attempt, wait_fixed @@ -95,8 +97,10 @@ class SummarizeCode(Action): return code_rsp async def run(self): - design_doc = await FileRepository.get_file(self.context.design_filename) - task_doc = await FileRepository.get_file(self.context.task_filename) + design_pathname = Path(self.context.design_filename) + design_doc = await FileRepository.get_file(filename=design_pathname.name, relative_path=design_pathname.parent) + task_pathname = Path(self.context.task_filename) + task_doc = await FileRepository.get_file(filename=task_pathname.name, relative_path=task_pathname.parent) src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace) code_blocks = [] for filename in self.context.codes_filenames: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 59ccb49a5..86cd24e33 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -19,7 +19,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action import Action -from metagpt.const import TEST_OUTPUTS_FILE_REPO +from metagpt.const import TEST_OUTPUTS_FILE_REPO, CODE_SUMMARIES_FILE_REPO from metagpt.logs import logger from metagpt.schema import CodingContext, RunCodeResult from metagpt.utils.common import CodeParser @@ -50,6 +50,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc # Debug logs ```text {logs} + +{summary_log} ``` ----- @@ -90,6 +92,8 @@ class WriteCode(Action): test_doc = await FileRepository.get_file( filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO ) + summary_doc = await FileRepository.get_file(filename=coding_context.design_doc.filename, + relative_path=CODE_SUMMARIES_FILE_REPO) logs = "" if test_doc: test_detail = RunCodeResult.loads(test_doc.content) @@ -100,6 +104,7 @@ class WriteCode(Action): code=coding_context.code_doc.content, logs=logs, filename=self.context.filename, + summary_log=summary_doc.content if summary_doc else "" ) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 3967a0578..eb89f1ad1 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -8,6 +8,7 @@ 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.2 of RFC 135, add incremental iteration functionality. 3. Move the document storage operations related to WritePRD from the save operation of WriteDesign. +@Modified By: mashenquan, 2023/12/5. Move the generation logic of the project name to WritePRD. """ from __future__ import annotations @@ -27,6 +28,7 @@ from metagpt.const import ( ) from metagpt.logs import logger from metagpt.schema import Document, Documents +from metagpt.utils.common import CodeParser from metagpt.utils.file_repository import FileRepository from metagpt.utils.get_template import get_template from metagpt.utils.mermaid import mermaid_to_file @@ -53,7 +55,7 @@ ATTENTION: Output carefully referenced "Format example" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "project_name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -85,9 +87,10 @@ and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] -{ +{{ "Language": "", "Original Requirements": "", + "project_name": "{project_name}", "Search Information": "", "Requirements": "", "Product Goals": [], @@ -111,7 +114,7 @@ and only output the json inside this tag, nothing else "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], "UI Design draft": "", "Anything UNCLEAR": "", -} +}} [/CONTENT] """, }, @@ -228,6 +231,7 @@ There are no unclear points. OUTPUT_MAPPING = { "Language": (str, ...), "Original Requirements": (str, ...), + "project_name": (str, ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), @@ -270,7 +274,7 @@ ATTENTION: Output carefully referenced "Old PRD" in format. {{ "Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc. "Original Requirements": "", # str, place the polished complete original requirements here - "project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc. + "project_name": "{project_name}", # str, if it's empty, name it with snake case style, like game_2048 / web_2048 / simple_crm etc. "Search Information": "", "Requirements": "", "Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals. @@ -320,6 +324,7 @@ class WritePRD(Action): if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"REWRITE PRD:{prd_doc.filename}") # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( @@ -327,6 +332,7 @@ class WritePRD(Action): ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc + logger.info(f"NEW PRD:{prd_doc.filename}") # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the # 'publish' message to transition the workflow to the next stage. This design allows room for global # optimization in subsequent steps. @@ -343,32 +349,36 @@ class WritePRD(Action): # logger.info(format) prompt_template, format_example = get_template(templates, format) + project_name = CONFIG.project_name if CONFIG.project_name else "" + format_example = format_example.format(project_name=project_name) # logger.info(prompt_template) # logger.info(format_example) prompt = prompt_template.format( - requirements=requirements, search_information=info, format_example=format_example + requirements=requirements, search_information=info, format_example=format_example, + project_name=project_name ) # logger.info(prompt) # prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + await self._rename_workspace(prd) return prd async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: - m = json.loads(old_prd_doc.content) - if m.get("Original Requirements") == new_requirement_doc.content: - # There have been no changes in the requirements, so they are considered unrelated. - return False prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) res = await self._aask(prompt=prompt) - logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + logger.info(f"REQ-RELATIVE:[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") if "YES" in res: return True return False async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: - prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content, + project_name=CONFIG.project_name) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) prd_doc.content = prd.instruct_content.json(ensure_ascii=False) + await self._rename_workspace(prd) return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: @@ -404,3 +414,19 @@ class WritePRD(Action): @staticmethod async def _save_pdf(prd_doc): await FileRepository.save_as(doc=prd_doc, with_suffix=".md", relative_path=PRD_PDF_FILE_REPO) + + @staticmethod + async def _rename_workspace(prd): + if CONFIG.project_path: # Updating on the old version has already been specified if it's valid. According to + # Section 2.2.3.10 of RFC 135 + if not CONFIG.project_name: + CONFIG.project_name = Path(CONFIG.project_path).name + return + + if not CONFIG.project_name: + if isinstance(prd, ActionOutput): + ws_name = prd.instruct_content.dict()["project_name"] + else: + ws_name = CodeParser.parse_str(block="project_name", text=prd) + CONFIG.project_name = ws_name + CONFIG.git_repo.rename_root(CONFIG.project_name) \ No newline at end of file diff --git a/metagpt/const.py b/metagpt/const.py index a646cea7a..bd735a5e1 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-11-1. According to Section 2.2.1 and 2.2.2 of RFC 116, added key definitions for common properties in the Message. @Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135. +@Modified By: mashenquan, 2023/12/5. Add directories for code summarization.. """ import contextvars import os @@ -87,5 +88,7 @@ PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" TEST_OUTPUTS_FILE_REPO = "test_outputs" +CODE_SUMMARIES_FILE_REPO = "docs/code_summaries" +CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries" YAPI_URL = "http://yapi.deepwisdomai.com/" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index d42835a1b..59279c402 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -13,6 +13,8 @@ @Modified By: mashenquan, 2023-11-27. 1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name. 2. According to the design in Section 2.2.3.5.5 of RFC 135, add incremental iteration functionality. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from __future__ import annotations @@ -23,7 +25,8 @@ from typing import Set from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO +from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, CODE_SUMMARIES_FILE_REPO, \ + CODE_SUMMARIES_PDF_FILE_REPO from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import ( @@ -33,6 +36,16 @@ from metagpt.schema import ( Documents, Message, ) +from metagpt.utils.common import any_to_str_set, any_to_str + +IS_PASS_PROMPT = """ +{context} + +---- +Does the above log indicate anything that needs to be done? +If there are any tasks to be completed, please answer 'NO' along with the to-do list in JSON format; +otherwise, answer 'YES' in JSON format. +""" class Engineer(Role): @@ -49,18 +62,18 @@ class Engineer(Role): """ def __init__( - self, - name: str = "Alex", - profile: str = "Engineer", - goal: str = "Write elegant, readable, extensible, efficient code", - constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", - n_borg: int = 1, - use_code_review: bool = False, + self, + name: str = "Alex", + profile: str = "Engineer", + goal: str = "Write elegant, readable, extensible, efficient code", + constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", + n_borg: int = 1, + use_code_review: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self.use_code_review = use_code_review - self._watch([WriteTasks]) + self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview]) self.code_todos = [] self.summarize_todos = [] self.n_borg = n_borg @@ -105,43 +118,87 @@ class Engineer(Role): if self._rc.todo is None: return None if isinstance(self._rc.todo, WriteCode): - changed_files = await self._act_sp_with_cr(review=self.use_code_review) - # Unit tests only. - if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files: - changed_files.add(CONFIG.REQA_FILENAME) - return Message( - content="\n".join(changed_files), - role=self.profile, - cause_by=WriteCodeReview if self.use_code_review else WriteCode, - send_to="Edward", # The name of QaEngineer - ) + return await self._act_write_code() if isinstance(self._rc.todo, SummarizeCode): - summaries = [] - for todo in self.summarize_todos: - summary = await todo.run() - summaries.append(summary.json(ensure_ascii=False)) + return await self._act_summarize() + return None + + async def _act_write_code(self): + changed_files = await self._act_sp_with_cr(review=self.use_code_review) + return Message( + content="\n".join(changed_files), + role=self.profile, + cause_by=WriteCodeReview if self.use_code_review else WriteCode, + send_to=self, + sent_from=self + ) + + async def _act_summarize(self): + code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO) + code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO) + tasks = [] + src_relative_path = CONFIG.src_workspace.relative_to(CONFIG.git_repo.workdir) + for todo in self.summarize_todos: + summary = await todo.run() + summary_filename = Path(todo.context.design_filename).with_suffix(".md").name + dependencies = {todo.context.design_filename, todo.context.task_filename} + for filename in todo.context.codes_filenames: + rpath = src_relative_path / filename + dependencies.add(str(rpath)) + await code_summaries_pdf_file_repo.save(filename=summary_filename, content=summary, + dependencies=dependencies) + is_pass, reason = await self._is_pass(summary) + if not is_pass: + todo.context.reason = reason + tasks.append(todo.context.dict()) + await code_summaries_file_repo.save(filename=Path(todo.context.design_filename).name, + content=todo.context.json(), dependencies=dependencies) + + if not tasks: return Message( - content="\n".join(summaries), + content="", role=self.profile, cause_by=SummarizeCode, - send_to=MESSAGE_ROUTE_TO_NONE, + sent_from=self, + send_to="Edward", # The name of QaEngineer ) - return None + return Message( + content=json.dumps(tasks), + role=self.profile, + cause_by=SummarizeCode, + send_to=self, + sent_from=self + ) + + async def _is_pass(self, summary) -> (str, str): + msgs = [{"role": "user", "content": IS_PASS_PROMPT.format(context=summary)}] + rsp = await self._llm.acompletion_text(messages=msgs, stream=False) + logger.info(rsp) + if "YES" in rsp: + return True, rsp + return False, rsp async def _think(self) -> Action | None: if not CONFIG.src_workspace: CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name - if not self.code_todos: - await self._new_code_actions() - elif not self.summarize_todos: - await self._new_summarize_actions() - else: + write_code_filters = any_to_str_set([WriteTasks, SummarizeCode]) + summarize_code_filters = any_to_str_set([WriteCode, WriteCodeReview]) + if not self._rc.news: return None - return self._rc.todo # For agent store + msg = self._rc.news[0] + if msg.cause_by in write_code_filters: + logger.info(f"TODO WriteCode:{msg.json()}") + await self._new_code_actions() + return self._rc.todo + if msg.cause_by in summarize_code_filters and msg.sent_from == any_to_str(self): + logger.info(f"TODO SummarizeCode:{msg.json()}") + await self._new_summarize_actions() + return self._rc.todo + return None @staticmethod async def _new_coding_context( - filename, src_file_repo, task_file_repo, design_file_repo, dependency + filename, src_file_repo, task_file_repo, design_file_repo, dependency ) -> CodingContext: old_code_doc = await src_file_repo.get(filename) if not old_code_doc: @@ -216,16 +273,16 @@ class Engineer(Role): async def _new_summarize_actions(self): src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) - changed_src_files = src_file_repo.changed_files + src_files = src_file_repo.all_files # Generate a SummarizeCode action for each pair of (system_design_doc, task_doc). summarizations = {} - for filename in changed_src_files: - dependencies = src_file_repo.get_dependency(filename=filename) + for filename in src_files: + dependencies = await src_file_repo.get_dependency(filename=filename) ctx = CodeSummarizeContext.loads(filenames=dependencies) if ctx not in summarizations: - summarizations[ctx] = set() + summarizations[ctx] = [] srcs = summarizations.get(ctx) - srcs.add(filename) + srcs.append(filename) for ctx, filenames in summarizations.items(): ctx.codes_filenames = filenames self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm)) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 41a3213dc..15a01b9e9 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -11,10 +11,13 @@ WriteTest/RunCode/DebugError object, rather than passing them in when calling the run function. 2. According to Section 2.2.3.5.7 of RFC 135, change the method of transferring files from using the Message to using file references. +@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results + of SummarizeCode. """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest # from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.summarize_code import SummarizeCode from metagpt.config import CONFIG from metagpt.const import ( MESSAGE_ROUTE_TO_NONE, @@ -40,13 +43,16 @@ class QaEngineer(Role): self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self._watch([SummarizeCode, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed async def _write_test(self, message: Message) -> None: - changed_files = message.content.splitlines() src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace) + changed_files = set(src_file_repo.changed_files.keys()) + # Unit tests only. + if CONFIG.reqa_file and CONFIG.reqa_file not in changed_files: + changed_files.add(CONFIG.reqa_file) tests_file_repo = CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO) for filename in changed_files: # write tests @@ -146,7 +152,7 @@ class QaEngineer(Role): ) return result_msg - code_filters = any_to_str_set({WriteCode, WriteCodeReview}) + code_filters = any_to_str_set({SummarizeCode}) test_filters = any_to_str_set({WriteTest, DebugError}) run_filters = any_to_str_set({RunCode}) for msg in self._rc.news: diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1e99cc1ff..2651be7eb 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -284,9 +284,10 @@ class Role: instruct_content=response.instruct_content, role=self.profile, cause_by=self._rc.todo, + sent_from=self, ) else: - msg = Message(content=response, role=self.profile, cause_by=self._rc.todo) + msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self) self._rc.memory.add(msg) return msg diff --git a/metagpt/schema.py b/metagpt/schema.py index d1174799a..51f395e65 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -324,10 +324,11 @@ class RunCodeResult(BaseModel): class CodeSummarizeContext(BaseModel): design_filename: str = "" task_filename: str = "" - codes_filenames: Set[str] = Field(default_factory=set) + codes_filenames: List[str] = Field(default_factory=list) + reason: str = "" @staticmethod - def loads(filenames: Set) -> CodeSummarizeContext: + def loads(filenames: List) -> CodeSummarizeContext: ctx = CodeSummarizeContext() for filename in filenames: if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO): @@ -337,3 +338,7 @@ class CodeSummarizeContext(BaseModel): ctx.task_filename = str(filename) continue return ctx + + def __hash__(self): + return hash((self.design_filename, self.task_filename)) + diff --git a/metagpt/utils/file_repository.py b/metagpt/utils/file_repository.py index 0815bf90a..a435a6b8e 100644 --- a/metagpt/utils/file_repository.py +++ b/metagpt/utils/file_repository.py @@ -151,6 +151,17 @@ class FileRepository: relative_files[str(rf)] = ct return relative_files + @property + def all_files(self) -> List: + """Get a dictionary of all files in the repository. + + The dictionary includes file paths relative to the current FileRepository. + + :return: A dictionary where keys are file paths and values are file information. + :rtype: List + """ + return self._git_repo.get_files(relative_path=self._relative_path) + def get_change_dir_files(self, dir: Path | str) -> List: """Get the files in a directory that have changed. diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index 7c9ec645f..090b7319d 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -11,7 +11,7 @@ from __future__ import annotations import shutil from enum import Enum from pathlib import Path -from typing import Dict +from typing import Dict, List from git.repo import Repo from git.repo.fun import is_git_dir @@ -200,6 +200,32 @@ class GitRepository: logger.info(f"Rename directory {str(self.workdir)} to {str(new_path)}") self._repository = Repo(new_path) + def get_files(self, relative_path: Path | str) -> List: + """Retrieve a list of files in the specified relative path. + + The method returns a list of file paths relative to the current FileRepository. + + :param relative_path: The relative path within the repository. + :type relative_path: Path or str + :return: A list of file paths in the specified directory. + :rtype: List[str] + """ + try: + relative_path = Path(relative_path).relative_to(self.workdir) + except ValueError: + relative_path = Path(relative_path) + + files = [] + try: + directory_path = Path(self.workdir) / relative_path + for file_path in directory_path.iterdir(): + if file_path.is_file(): + rpath = file_path.relative_to(directory_path) + files.append(str(rpath)) + except Exception as e: + logger.error(f"Error: {e}") + return files + if __name__ == "__main__": path = DEFAULT_WORKSPACE_ROOT / "git"