From 882f22da352f8099af6fc0974a292c4866cb6c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 24 Nov 2023 19:56:27 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=B5=81=E7=A8=8B=E8=B0=83=E9=80=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/debug_error.py | 45 +++++++++---- metagpt/actions/design_api.py | 93 +++++++++++++-------------- metagpt/actions/project_management.py | 41 +++++++++++- metagpt/actions/run_code.py | 29 +++++---- metagpt/actions/write_code.py | 69 +++++++++++--------- metagpt/actions/write_prd.py | 74 +++++++++++++++++++-- metagpt/const.py | 2 +- metagpt/roles/engineer.py | 8 ++- metagpt/roles/qa_engineer.py | 26 ++++---- metagpt/schema.py | 14 ++++ metagpt/utils/git_repository.py | 9 ++- 11 files changed, 274 insertions(+), 136 deletions(-) diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 7a12e18f8..d0c3652b4 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -8,7 +8,10 @@ import re from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import TEST_CODES_FILE_REPO, TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger +from metagpt.schema import RunCodeResult from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ @@ -19,7 +22,20 @@ Based on the message, first, figure out your own role, i.e. Engineer or QaEngine then rewrite the development code or the test code based on your role, the error, and the summary, such that all bugs are fixed and the code performs well. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the test case or script and triple quotes. The message is as follows: -{context} +# Legacy Code +```python +{code} +``` +--- +# Unit Test Code +```python +{test_code} +``` +--- +# Console logs +```text +{logs} +``` --- Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. @@ -30,25 +46,26 @@ class DebugError(Action): def __init__(self, name="DebugError", context=None, llm=None): super().__init__(name, context, llm) - # async def run(self, code, error): - # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ - # f"\n\n{error}\n\nPlease try to fix the error in this code." - # fixed_code = await self._aask(prompt) - # return fixed_code - async def run(self, *args, **kwargs) -> str: + output_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get(self.context.output_filename) + if not output_doc: + return "" + output_detail = RunCodeResult.loads(output_doc.content) pattern = r"Ran (\d+) tests in ([\d.]+)s\n\nOK" - matches = re.search(pattern, self.context.output) + matches = re.search(pattern, output_detail.stderr) if matches: - return "", "the original code works fine, no need to debug" + return "" - file_name = self.context.code_filename - logger.info(f"Debug and rewrite {file_name}") - - prompt = PROMPT_TEMPLATE.format(context=self.context.output) + logger.info(f"Debug and rewrite {self.context.code_filename}") + code_doc = await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).get(self.context.code_filename) + if not code_doc: + return "" + test_doc = await CONFIG.git_repo.new_file_repository(TEST_CODES_FILE_REPO).get(self.context.test_filename) + if not test_doc: + return "" + prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr) rsp = await self._aask(prompt) - code = CodeParser.parse_code(block="", text=rsp) return code diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index a8f89473d..02f87bc47 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -6,7 +6,6 @@ @File : design_api.py """ import json -import shutil from pathlib import Path from typing import List @@ -18,13 +17,11 @@ from metagpt.const import ( SEQ_FLOW_FILE_REPO, SYSTEM_DESIGN_FILE_REPO, SYSTEM_DESIGN_PDF_FILE_REPO, - WORKSPACE_ROOT, ) from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template -from metagpt.utils.json_to_markdown import json_to_markdown from metagpt.utils.mermaid import mermaid_to_file templates = { @@ -157,6 +154,34 @@ OUTPUT_MAPPING = { "Anything UNCLEAR": (str, ...), } +MERGE_PROMPT = """ +## Old Design +{old_design} + +## Context +{context} + +----- +Role: You are an architect; The goal is to incrementally update the "Old Design" based on the information provided by the "Context," aiming to design a state-of-the-art (SOTA) Python system compliant with PEP8. Additionally, the objective is to optimize the use of high-quality open-source tools. +Requirement: Fill in the following missing information based on the context, each section name is a key in json +Max Output: 8192 chars or 2048 tokens. Try to use them up. + +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. + +## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores + +## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here + +## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. + +## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Design" format, +and only output the json inside this tag, nothing else +""" + class WriteDesign(Action): def __init__(self, name, context=None, llm=None): @@ -167,50 +192,6 @@ class WriteDesign(Action): "clearly and in detail." ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # Folder does not exist, but we don't care - workspace.mkdir(parents=True, exist_ok=True) - - async def _save_prd(self, docs_path, resources_path, context): - prd_file = docs_path / "prd.md" - if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") - - if context[-1].instruct_content: - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) - - async def _save_system_design(self, docs_path, resources_path, system_design): - data_api_design = system_design.instruct_content.dict()[ - "Data structures and interface definitions" - ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) - seq_flow = system_design.instruct_content.dict()[ - "Program call flow" - ] # CodeParser.parse_code(block="Program call flow", text=content) - await mermaid_to_file(data_api_design, resources_path / "data_api_design") - await mermaid_to_file(seq_flow, resources_path / "seq_flow") - system_design_file = docs_path / "system_design.md" - logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) - - async def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - ws_name = system_design.instruct_content.dict()["Python package name"] - else: - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / "docs" - resources_path = workspace / "resources" - docs_path.mkdir(parents=True, exist_ok=True) - resources_path.mkdir(parents=True, exist_ok=True) - await self._save_prd(docs_path, resources_path, context) - await self._save_system_design(docs_path, resources_path, system_design) - async def run(self, with_messages, format=CONFIG.prompt_format): # 通过git diff来识别docs/prds下哪些PRD文档发生了变动 prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -234,7 +215,8 @@ class WriteDesign(Action): filename=filename, prds_file_repo=prds_file_repo, system_design_file_repo=system_design_file_repo ) changed_files.docs[filename] = doc - + if not changed_files.docs: + logger.info("Nothing has changed.") # 等docs/system_designs/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=changed_files.json(), instruct_content=changed_files) @@ -253,10 +235,21 @@ class WriteDesign(Action): await self._rename_workspace(system_design) return system_design - async def _merge(self, prd_doc, system_design_doc): + async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format): + prompt = MERGE_PROMPT.format(old_design=system_design_doc.content, context=prd_doc.content) + system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python + # package name" contain space, have to use setattr + setattr( + system_design.instruct_content, + "Python package name", + system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), + ) + system_design_doc.content = system_design.instruct_content.json() return system_design_doc - async def _rename_workspace(self, system_design): + @staticmethod + async def _rename_workspace(system_design): if CONFIG.WORKDIR: # 已经指定了在旧版本上更新 return diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 686aa3689..4fd944027 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -17,6 +17,7 @@ from metagpt.const import ( TASK_PDF_FILE_REPO, WORKSPACE_ROOT, ) +from metagpt.logs import logger from metagpt.schema import Document, Documents from metagpt.utils.common import CodeParser from metagpt.utils.get_template import get_template @@ -169,6 +170,35 @@ OUTPUT_MAPPING = { "Anything UNCLEAR": (str, ...), } +MERGE_PROMPT = """ +# Context +{context} + +## Old Tasks +{old_tasks} +----- +Role: You are a project manager; The goal is to merge the new PRD/technical design content from 'Context' into 'Old Tasks.' Based on this merged result, break down tasks, give a task list, and analyze task dependencies to start with the prerequisite modules. +Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. + +## Required Python third-party packages: Provided in requirements.txt format + +## Required Other language third-party packages: Provided in requirements.txt format + +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + +## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first + +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first + +## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old Tasks" format, +and only output the json inside this tag, nothing else +""" + class WriteTasks(Action): def __init__(self, name="CreateTasks", context=None, llm=None): @@ -209,6 +239,8 @@ class WriteTasks(Action): ) change_files.docs[filename] = task_doc + if not change_files.docs: + logger.info("Nothing has changed.") # 等docs/tasks/下所有文件都处理完才发publish message，给后续做全局优化留空间。 return ActionOutput(content=change_files.json(), instruct_content=change_files) @@ -216,7 +248,7 @@ class WriteTasks(Action): system_design_doc = await system_design_file_repo.get(filename) task_doc = await tasks_file_repo.get(filename) if task_doc: - task_doc = await self._merge(system_design_doc=system_design_doc, task_dock=task_doc) + task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc) else: rsp = await self._run_new_tasks(context=system_design_doc.content) task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json()) @@ -234,8 +266,11 @@ class WriteTasks(Action): # self._save(context, rsp) return rsp - async def _merge(self, system_design_doc, task_dock) -> Document: - return task_dock + async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document: + prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content) + rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) + task_doc.content = rsp.instruct_content.json() + return task_doc @staticmethod async def _update_requirements(doc): diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index b244577a7..242eaa25d 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -12,6 +12,7 @@ from typing import Tuple from metagpt.actions.action import Action from metagpt.logs import logger +from metagpt.schema import RunCodeResult PROMPT_TEMPLATE = """ Role: You are a senior development and qa engineer, your role is summarize the code running result. @@ -89,14 +90,7 @@ class RunCode(Action): additional_python_paths = [working_directory] + additional_python_paths additional_python_paths = ":".join(additional_python_paths) env["PYTHONPATH"] = additional_python_paths + ":" + env.get("PYTHONPATH", "") - - install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] - logger.info(" ".join(install_command)) - subprocess.run(install_command, check=True, cwd=working_directory, env=env) - - install_pytest_command = ["python", "-m", "pip", "install", "pytest"] - logger.info(" ".join(install_pytest_command)) - subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + RunCode._install_dependencies(working_directory=working_directory, env=env) # Start the subprocess process = subprocess.Popen( @@ -113,7 +107,7 @@ class RunCode(Action): stdout, stderr = process.communicate() return stdout.decode("utf-8"), stderr.decode("utf-8") - async def run(self, *args, **kwargs) -> str: + async def run(self, *args, **kwargs) -> RunCodeResult: logger.info(f"Running {' '.join(self.context.command)}") if self.context.mode == "script": outs, errs = await self.run_script( @@ -139,7 +133,20 @@ class RunCode(Action): prompt = PROMPT_TEMPLATE.format(context=context) rsp = await self._aask(prompt) + return RunCodeResult(summary=rsp, stdout=outs, stderr=errs) - result = context + rsp + @staticmethod + def _install_dependencies(working_directory, env): + install_command = ["python", "-m", "pip", "install", "-r", "requirements.txt"] + logger.info(" ".join(install_command)) + try: + subprocess.run(install_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") - return result + install_pytest_command = ["python", "-m", "pip", "install", "pytest"] + logger.info(" ".join(install_pytest_command)) + try: + subprocess.run(install_pytest_command, check=True, cwd=working_directory, env=env) + except subprocess.CalledProcessError as e: + logger.warning(f"{e}") diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index d4d33fe0c..c9b6c3b9e 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,16 +7,15 @@ @Modified By: mashenquan, 2023-11-1. In accordance with Chapter 2.1.3 of RFC 116, modify the data type of the `cause_by` value of the `Message` object. """ -import json from tenacity import retry, stop_after_attempt, wait_fixed -from metagpt.actions import WriteDesign from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.const import TEST_OUTPUTS_FILE_REPO from metagpt.logs import logger -from metagpt.schema import CodingContext -from metagpt.utils.common import CodeParser, any_to_str +from metagpt.schema import CodingContext, RunCodeResult +from metagpt.utils.common import CodeParser PROMPT_TEMPLATE = """ NOTICE @@ -33,8 +32,25 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc 7. Do not use public member functions that do not exist in your design. ----- -# Context -{context} +# Design +```json +{design} +``` +----- +# Tasks +```json +{tasks} +``` +----- +# Legacy Code +```python +{code} +``` +----- +# Debug logs +```text +{logs} +``` ----- ## Format example ----- @@ -51,26 +67,6 @@ class WriteCode(Action): def __init__(self, name="WriteCode", context=None, llm=None): super().__init__(name, context, llm) - def _is_invalid(self, filename): - return any(i in filename for i in ["mp3", "wav"]) - - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == any_to_str(WriteDesign)][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt) -> str: code_rsp = await self._aask(prompt) @@ -78,12 +74,21 @@ class WriteCode(Action): return code async def run(self, *args, **kwargs) -> CodingContext: - m = json.loads(self.context.content) - coding_context = CodingContext(**m) - context = "\n".join( - [coding_context.design_doc.content, coding_context.task_doc.content, coding_context.code_doc.content] + coding_context = CodingContext.loads(self.context.content) + test_doc = await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).get( + "test_" + coding_context.filename + ".json" + ) + logs = "" + if test_doc: + test_detail = RunCodeResult.loads(test_doc.content) + logs = test_detail.stderr + prompt = PROMPT_TEMPLATE.format( + design=coding_context.design_doc.content, + tasks=coding_context.task_doc.content, + code=coding_context.code_doc.content, + logs=logs, + filename=self.context.filename, ) - prompt = PROMPT_TEMPLATE.format(context=context, filename=self.context.filename) logger.info(f"Writing {coding_context.filename}..") code = await self.write_code(prompt) coding_context.code_doc.content = code diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 8b03ac29a..532f5bc34 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -219,6 +219,7 @@ There are no unclear points. }, } + OUTPUT_MAPPING = { "Original Requirements": (str, ...), "Product Goals": (List[str], ...), @@ -231,13 +232,60 @@ OUTPUT_MAPPING = { "Anything UNCLEAR": (str, ...), } +IS_RELATIVE_PROMPT = """ +## PRD: +{old_prd} + +## New Requirement: +{requirements} + +___ +You are a professional product manager; You need to assess whether the new requirements are relevant to the existing PRD to determine whether to merge the new requirements into this PRD. +Is the newly added requirement in "New Requirement" related to the PRD? +Respond with `YES` if it is related, `NO` if it is not, and provide the reasons. Return the response in JSON format. +""" + +MERGE_PROMPT = """ +# Context +## Original Requirements +{requirements} + + +## Old PRD +{old_prd} +----- +Role: You are a professional product manager; The goal is to merge the newly added requirements into the existing PRD in order to design a concise, usable, and efficient product. +Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design + +## Original Requirements: Provide as Plain text, place the polished complete original requirements here + +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple + +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less + +## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible + +## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + +## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. + +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower + +## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like "Old PRD" format, +and only output the json inside this tag, nothing else +""" + class WritePRD(Action): def __init__(self, name="", context=None, llm=None): super().__init__(name, context, llm) async def run(self, with_messages, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: - # 判断哪些需求文档需要重写：调LLM判断新增需求与prd是否相关，若相关就rewrite prd + # Determine which requirement documents need to be rewritten: Use LLM to assess whether new requirements are + # related to the PRD. If they are related, rewrite the PRD. docs_file_repo = CONFIG.git_repo.new_file_repository(DOCS_FILE_REPO) requirement_doc = await docs_file_repo.get(REQUIREMENT_FILENAME) prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO) @@ -250,14 +298,16 @@ class WritePRD(Action): if not prd_doc: continue change_files.docs[prd_doc.filename] = prd_doc - # 如果没有任何PRD，就使用docs/requirement.txt生成一个prd + # If there is no existing PRD, generate one using 'docs/requirement.txt'. if not change_files.docs: prd_doc = await self._update_prd( requirement_doc=requirement_doc, prd_doc=None, prds_file_repo=prds_file_repo, *args, **kwargs ) if prd_doc: change_files.docs[prd_doc.filename] = prd_doc - # 等docs/prds/下所有文件都与新增需求对比完后，再触发publish message让工作流跳转到下一环节。如此设计是为了给后续做全局优化留空间。 + # Once all files under 'docs/prds/' have been compared with the newly added requirements, trigger the + # 'publish' message to transition the workflow to the next stage. This design allows room for global + # optimization in subsequent steps. return ActionOutput(content=change_files.json(), instruct_content=change_files) async def _run_new_requirement(self, requirements, format=CONFIG.prompt_format, *args, **kwargs) -> ActionOutput: @@ -278,11 +328,23 @@ class WritePRD(Action): prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) return prd - async def _is_relative_to(self, doc1, doc2) -> bool: + async def _is_relative_to(self, new_requirement_doc, old_prd_doc) -> bool: + m = json.loads(old_prd_doc.content) + if m.get("Original Requirements") == new_requirement_doc.content: + # There have been no changes in the requirements, so they are considered unrelated. + return False + prompt = IS_RELATIVE_PROMPT.format(old_prd=old_prd_doc.content, requirements=new_requirement_doc.content) + res = await self._aask(prompt=prompt) + logger.info(f"[{new_requirement_doc.root_relative_path}, {old_prd_doc.root_relative_path}]: {res}") + if "YES" in res: + return True return False - async def _merge(self, doc1, doc2) -> Document: - pass + async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document: + prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content) + prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format) + prd_doc.content = prd.instruct_content.json() + return prd_doc async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None: if not prd_doc: diff --git a/metagpt/const.py b/metagpt/const.py index 311712013..49965b622 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -63,4 +63,4 @@ SYSTEM_DESIGN_PDF_FILE_REPO = "resources/system_design" PRD_PDF_FILE_REPO = "resources/prd" TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks" TEST_CODES_FILE_REPO = "tests" -OUTPUTS_FILE_REPO = "outputs" +TEST_OUTPUTS_FILE_REPO = "test_outputs" diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 89827a1ca..b6ecc4767 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -90,6 +90,8 @@ class Engineer(Role): self._rc.memory.add(msg) changed_files.add(coding_context.code_doc.filename) + if not changed_files: + logger.info("Nothing has changed.") return changed_files async def _act(self) -> Message: @@ -136,8 +138,8 @@ class Engineer(Role): root_path=str(src_file_repo.root_path), filename=task_filename, content=context.json() ) if task_filename in changed_files.docs: - logger.error( - f"Log to expose potential file name conflicts: {coding_doc.json()} & " + logger.warning( + f"Log to expose potential conflicts: {coding_doc.json()} & " f"{changed_files.docs[task_filename].json()}" ) changed_files.docs[task_filename] = coding_doc @@ -168,7 +170,7 @@ class Engineer(Role): old_code_doc = await src_file_repo.get(filename) if not old_code_doc: old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="") - dependencies = {Path(i) for i in dependency.get(old_code_doc.root_relative_path)} + dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)} task_doc = None design_doc = None for i in dependencies: diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 68138d925..a88b01e37 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -9,7 +9,11 @@ """ from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest from metagpt.config import CONFIG -from metagpt.const import MESSAGE_ROUTE_TO_NONE, OUTPUTS_FILE_REPO, TEST_CODES_FILE_REPO +from metagpt.const import ( + MESSAGE_ROUTE_TO_NONE, + TEST_CODES_FILE_REPO, + TEST_OUTPUTS_FILE_REPO, +) from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Document, Message, RunCodeContext, TestingContext @@ -86,20 +90,17 @@ class QaEngineer(Role): return run_code_context.code = src_doc.content run_code_context.test_code = test_doc.content - result_msg = await RunCode(context=run_code_context, llm=self._llm).run() - run_code_context.output_filename = run_code_context.test_filename + ".md" - await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).save( + result = await RunCode(context=run_code_context, llm=self._llm).run() + run_code_context.output_filename = run_code_context.test_filename + ".json" + await CONFIG.git_repo.new_file_repository(TEST_OUTPUTS_FILE_REPO).save( filename=run_code_context.output_filename, - content=result_msg, + content=result.json(), dependencies={src_doc.root_relative_path, test_doc.root_relative_path}, ) run_code_context.code = None run_code_context.test_code = None - recipient = parse_recipient(result_msg) # the recipient might be Engineer or myself - mappings = { - "Engineer": "Alex", - "QaEngineer": "Edward", - } + recipient = parse_recipient(result.summary) # the recipient might be Engineer or myself + mappings = {"Engineer": "Alex", "QaEngineer": "Edward"} self.publish_message( Message( content=run_code_context.json(), @@ -112,16 +113,11 @@ class QaEngineer(Role): async def _debug_error(self, msg): run_code_context = RunCodeContext.loads(msg.content) - output_doc = await CONFIG.git_repo.new_file_repository(OUTPUTS_FILE_REPO).get(run_code_context.output_filename) - if not output_doc: - return - run_code_context.output = output_doc.content code = await DebugError(context=run_code_context, llm=self._llm).run() await CONFIG.git_repo.new_file_repository(CONFIG.src_workspace).save( filename=run_code_context.code_filename, content=code ) run_code_context.output = None - run_code_context.output_filename = None self.publish_message( Message( content=run_code_context.json(), diff --git a/metagpt/schema.py b/metagpt/schema.py index 53a22f0e6..e910fc866 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -295,3 +295,17 @@ class RunCodeContext(BaseModel): return RunCodeContext(**m) except Exception: return None + + +class RunCodeResult(BaseModel): + summary: str + stdout: str + stderr: str + + @staticmethod + def loads(val: str) -> RunCodeResult | None: + try: + m = json.loads(val) + return RunCodeResult(**m) + except Exception: + return None diff --git a/metagpt/utils/git_repository.py b/metagpt/utils/git_repository.py index ace0cf8a2..b8e35199b 100644 --- a/metagpt/utils/git_repository.py +++ b/metagpt/utils/git_repository.py @@ -72,7 +72,14 @@ class GitRepository: :param local_path: The local path where the new Git repository will be initialized. """ - self._repository = Repo.init(path=local_path) + self._repository = Repo.init(path=Path(local_path)) + + gitignore_filename = Path(local_path) / ".gitignore" + ignores = ["__pycache__", "*.pyc"] + with open(str(gitignore_filename), mode="w") as writer: + writer.write("\n".join(ignores)) + self._repository.index.add([".gitignore"]) + self._repository.index.commit("Add .gitignore") def add_change(self, files: Dict): """Add or remove files from the staging area based on the provided changes.