From 8a73d5cadbe40f7b275ddba37f680a141259756c Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Tue, 21 Nov 2023 14:38:19 +0800 Subject: [PATCH 1/6] add increment development function --- metagpt/actions/__init__.py | 1 + metagpt/actions/refine.py | 10 + metagpt/actions/refine_design_api.py | 231 ++++++++++++++++ metagpt/actions/refine_prd.py | 260 +++++++++++++++++++ metagpt/actions/refine_project_management.py | 208 +++++++++++++++ metagpt/actions/write_code.py | 10 +- metagpt/roles/architect.py | 48 +++- metagpt/roles/engineer.py | 74 +++++- metagpt/roles/product_manager.py | 36 ++- metagpt/roles/project_manager.py | 38 ++- metagpt/roles/role.py | 1 + startup.py | 54 +++- 12 files changed, 943 insertions(+), 28 deletions(-) create mode 100644 metagpt/actions/refine.py create mode 100644 metagpt/actions/refine_design_api.py create mode 100644 metagpt/actions/refine_prd.py create mode 100644 metagpt/actions/refine_project_management.py diff --git a/metagpt/actions/__init__.py b/metagpt/actions/__init__.py index b004bd58e..a9087e822 100644 --- a/metagpt/actions/__init__.py +++ b/metagpt/actions/__init__.py @@ -15,6 +15,7 @@ from metagpt.actions.design_api import WriteDesign from metagpt.actions.design_api_review import DesignReview from metagpt.actions.design_filenames import DesignFilenames from metagpt.actions.project_management import AssignTasks, WriteTasks +from metagpt.actions.refine import Refine from metagpt.actions.research import CollectLinks, WebBrowseAndSummarize, ConductResearch from metagpt.actions.run_code import RunCode from metagpt.actions.search_and_summarize import SearchAndSummarize diff --git a/metagpt/actions/refine.py b/metagpt/actions/refine.py new file mode 100644 index 000000000..beea40fc8 --- /dev/null +++ b/metagpt/actions/refine.py @@ -0,0 +1,10 @@ +from metagpt.actions import Action + + +# 增量开发动作的基类 +class Refine(Action): + def __init__(self, name="Refine", context=None, llm=None): + super().__init__(name, context, llm) + + def run(self, *args, **kwargs): + raise NotImplementedError diff --git a/metagpt/actions/refine_design_api.py b/metagpt/actions/refine_design_api.py new file mode 100644 index 000000000..591db175a --- /dev/null +++ b/metagpt/actions/refine_design_api.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +import shutil +from pathlib import Path +from typing import List + +from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger +from metagpt.utils.common import CodeParser +from metagpt.utils.get_template import get_template +from metagpt.utils.json_to_markdown import json_to_markdown +from metagpt.utils.mermaid import mermaid_to_file + +templates = { + "json": { + "PROMPT_TEMPLATE": """ +# Context +{context} + +## Legacy Design +{legacy} + +## Format example +{format_example} +----- +Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Requirement: Fill in the following missing information based on the context, each section name is a key in json +Max Output: 8192 chars or 2048 tokens. Try to use them up. + +## Difference Description: Provided as Python list[str], the list of differences between the new design and the legacy design + +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. + +## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores + +## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here + +## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. + +## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, +and only output the json inside this tag, nothing else +""", + "FORMAT_EXAMPLE": """ +[CONTENT] +{ + "Difference Description": ["The ..."], + "Implementation approach": "We will ...", + "Python package name": "snake_game", + "File list": ["main.py"], + "Data structures and interface definitions": ' + classDiagram + class Game{ + +int score + } + ... + Game "1" -- "1" Food: has + ', + "Program call flow": ' + sequenceDiagram + participant M as Main + ... + G->>M: end game + ', + "Anything UNCLEAR": "The requirement is clear to me." +} +[/CONTENT] +""", + }, + "markdown": { + "PROMPT_TEMPLATE": """ +# Context +{context} + +## Legacy Design +{legacy} + +## Format example +{format_example} +----- +Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately +Max Output: 8192 chars or 2048 tokens. Try to use them up. +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. + +## Difference Description: Provided as Python list[str], the list of differences between the new design and the legacy design + +## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. + +## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores + +## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here + +## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. + +## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +""", + "FORMAT_EXAMPLE": """ +--- +## Difference Description +```python +[ + "The ...", +] +``` + +## Implementation approach +We will ... + +## Python package name +```python +"snake_game" +``` + +## File list +```python +[ + "main.py", +] +``` + +## Data structures and interface definitions +```mermaid +classDiagram + class Game{ + +int score + } + ... + Game "1" -- "1" Food: has +``` + +## Program call flow +```mermaid +sequenceDiagram + participant M as Main + ... + G->>M: end game +``` + +## Anything UNCLEAR +The requirement is clear to me. +--- +""", + }, +} + +OUTPUT_MAPPING = { + "Difference Description": (List[str], ...), + "Implementation approach": (str, ...), + "Python package name": (str, ...), + "File list": (List[str], ...), + "Data structures and interface definitions": (str, ...), + "Program call flow": (str, ...), + "Anything UNCLEAR": (str, ...), +} + + +class RefineDesign(Action): + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) + self.desc = ( + "Based on the PRD, think about the system design, and design the corresponding APIs, " + "data structures, library tables, processes, and paths. Please provide your design, feedback " + "clearly and in detail." + ) + + def recreate_workspace(self, workspace: Path): + try: + shutil.rmtree(workspace) + except FileNotFoundError: + pass # Folder does not exist, but we don't care + workspace.mkdir(parents=True, exist_ok=True) + + async def _save_prd(self, docs_path, resources_path, context): + prd_file = docs_path / "prd.md" + if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: + quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] + await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") + + if context[-1].instruct_content: + logger.info(f"Saving PRD to {prd_file}") + prd_file.write_text(json_to_markdown(context[-1].instruct_content.dict())) + + async def _save_system_design(self, docs_path, resources_path, system_design): + data_api_design = system_design.instruct_content.dict()[ + "Data structures and interface definitions" + ] # CodeParser.parse_code(block="Data structures and interface definitions", text=content) + seq_flow = system_design.instruct_content.dict()[ + "Program call flow" + ] # CodeParser.parse_code(block="Program call flow", text=content) + await mermaid_to_file(data_api_design, resources_path / "data_api_design") + await mermaid_to_file(seq_flow, resources_path / "seq_flow") + system_design_file = docs_path / "system_design.md" + logger.info(f"Saving System Designs to {system_design_file}") + system_design_file.write_text((json_to_markdown(system_design.instruct_content.dict()))) + + async def _save(self, context, system_design): + if isinstance(system_design, ActionOutput): + ws_name = system_design.instruct_content.dict()["Python package name"] + else: + ws_name = CodeParser.parse_str(block="Python package name", text=system_design) + workspace = WORKSPACE_ROOT / ws_name + self.recreate_workspace(workspace) + docs_path = workspace / "docs" + resources_path = workspace / "resources" + docs_path.mkdir(parents=True, exist_ok=True) + resources_path.mkdir(parents=True, exist_ok=True) + await self._save_prd(docs_path, resources_path, context) + await self._save_system_design(docs_path, resources_path, system_design) + + async def run(self, context, legacy, format=CONFIG.prompt_format): + prompt_template, format_example = get_template(templates, format) + prompt = prompt_template.format(context=context, legacy=legacy, format_example=format_example) + # system_design = await self._aask(prompt) + system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format) + # fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python package name" contain space, have to use setattr + setattr( + system_design.instruct_content, + "Python package name", + system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'), + ) + await self._save(context, system_design) + return system_design diff --git a/metagpt/actions/refine_prd.py b/metagpt/actions/refine_prd.py new file mode 100644 index 000000000..1e2709c6b --- /dev/null +++ b/metagpt/actions/refine_prd.py @@ -0,0 +1,260 @@ +from typing import List + +from metagpt.actions import Refine, ActionOutput, SearchAndSummarize +from metagpt.config import CONFIG +from metagpt.logs import logger +from metagpt.utils.get_template import get_template + +increment_template = { + "json": { + "PROMPT_TEMPLATE": """ +# Context +## User's New Requirements +{new_requirements} + +## Difference Description +{difference_description} + +## Legacy PRD +{legacy} + +## Search Information +{search_information} + +## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME +```mermaid +quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + "Campaign: A": [0.3, 0.6] + "Campaign B": [0.45, 0.23] + "Campaign C": [0.57, 0.69] + "Campaign D": [0.78, 0.34] + "Campaign E": [0.40, 0.34] + "Campaign F": [0.35, 0.78] + "Our Target Product": [0.5, 0.6] +``` + +## Format example +{format_example} +----- +Role: You are a professional product manager; the goal is to design a concise, usable, efficient product based on the new requirements, the difference description and Legacy PRD. +Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design + +## New Requirements: Provide as Plain text and place the new requirements here + +## Difference Description: Provide as Python list[str], up to 5 clear, difference descriptions. If the requirement itself is simple, the difference description should also be simple + +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple + +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less + +## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible + +## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + +## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. + +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower + +## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. +## Anything UNCLEAR: Provide as Plain text. Make clear here. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, +and only output the json inside this tag, nothing else +""", + "FORMAT_EXAMPLE": """ +[CONTENT] +{ + "New Requirements": "", + "Difference Description": "", + "Search Information": "", + "Requirements": "", + "Product Goals": [], + "User Stories": [], + "Competitive Analysis": [], + "Competitive Quadrant Chart": "quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + Campaign A: [0.3, 0.6] + Campaign B: [0.45, 0.23] + Campaign C: [0.57, 0.69] + Campaign D: [0.78, 0.34] + Campaign E: [0.40, 0.34] + Campaign F: [0.35, 0.78]", + "Requirement Analysis": "", + "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], + "UI Design draft": "", + "Anything UNCLEAR": "", +} +[/CONTENT] +""", + }, + "markdown": { + "PROMPT_TEMPLATE": """ +# Context +You need to refine the requirements based on the new requirements and the existing requirements' output. +## User's New Requirements +{new_requirements} + +## Difference Description +{difference_description} + +## Legacy PRD +{legacy} + +## Search Information +{search_information} + +## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME +```mermaid +quadrantChart + title Reach and engagement of campaigns + x-axis Low Reach --> High Reach + y-axis Low Engagement --> High Engagement + quadrant-1 We should expand + quadrant-2 Need to promote + quadrant-3 Re-evaluate + quadrant-4 May be improved + "Campaign: A": [0.3, 0.6] + "Campaign B": [0.45, 0.23] + "Campaign C": [0.57, 0.69] + "Campaign D": [0.78, 0.34] + "Campaign E": [0.40, 0.34] + "Campaign F": [0.35, 0.78] + "Our Target Product": [0.5, 0.6] +``` + +## Format example +{format_example} +----- +Role: You are a professional product manager; the goal is to design a concise, usable, efficient product +Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. + +## New Requirements: Provide as Plain text and place the new requirements here + +## Difference Description: Provide as Python list[str], up to 5 clear, difference descriptions. If the requirement itself is simple, the difference description should also be simple + +## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple + +## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less + +## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible + +## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. + +## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. + +## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower + +## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. +## Anything UNCLEAR: Provide as Plain text. Make clear here. +""", + "FORMAT_EXAMPLE": """ +--- +## New Requirements +The boss ... + +## Difference Description +```python +[ + "...", +] + +## Product Goals +```python +[ + "Create a ...", +] +``` + +## User Stories +```python +[ + "As a user, ...", +] +``` + +## Competitive Analysis +```python +[ + "Python Snake Game: ...", +] +``` + +## Competitive Quadrant Chart +```mermaid +quadrantChart + title Reach and engagement of campaigns + ... + "Our Target Product": [0.6, 0.7] +``` + +## Requirement Analysis +The product should be a ... + +## Requirement Pool +```python +[ + ["End game ...", "P0"] +] +``` + +## UI Design draft +Give a basic function description, and a draft + +## Anything UNCLEAR +There are no unclear points. +--- +""", + }, +} + +INCREMENT_OUTPUT_MAPPING = { + "New Requirements": (str, ...), + # "Major Enhancements": (List[str], ...), + "Difference Description": (List[str], ...), + "Product Goals": (List[str], ...), + "User Stories": (List[str], ...), + "Competitive Analysis": (List[str], ...), + "Competitive Quadrant Chart": (str, ...), + "Requirement Analysis": (str, ...), + "Requirement Pool": (List[List[str]], ...), + "UI Design draft": (str, ...), + "Anything UNCLEAR": (str, ...), +} + + +# 对于产品经理，增量开发的动作是：RefinePDR，输出是结合新需求和已有需求输出的新的PDR +class RefinePRD(Refine): + + def __init__(self, name="RefinePRD", context=None, llm=None): + super().__init__(name, context, llm) + + async def run(self, new_requirements, difference_description, legacy, format=CONFIG.prompt_format, *args, **kwargs): + sas = SearchAndSummarize() + rsp = "" + info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" + if sas.result: + logger.info(sas.result) + logger.info(rsp) + + prompt_template, format_example = get_template(increment_template, format) + prompt = prompt_template.format( + new_requirements=new_requirements, difference_description=difference_description, legacy=legacy, search_information=info, + format_example=format_example + ) + logger.debug(prompt) + prd = await self._aask_v1(prompt, "prd", INCREMENT_OUTPUT_MAPPING, format=format) + return prd diff --git a/metagpt/actions/refine_project_management.py b/metagpt/actions/refine_project_management.py new file mode 100644 index 000000000..354996f3e --- /dev/null +++ b/metagpt/actions/refine_project_management.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +from typing import List + +from metagpt.actions.action import Action +from metagpt.config import CONFIG +from metagpt.const import WORKSPACE_ROOT +from metagpt.utils.common import CodeParser +from metagpt.utils.get_template import get_template +from metagpt.utils.json_to_markdown import json_to_markdown + +templates = { + "json": { + "PROMPT_TEMPLATE": """ +# Context +{context} + +## Legacy Design +{legacy} + +## Format example +{format_example} +----- +Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. + +## Required Python third-party packages: Provided in requirements.txt format + +## Required Other language third-party packages: Provided in requirements.txt format + +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + +## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first + +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first + +## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. + +## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. + +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, +and only output the json inside this tag, nothing else +""", + "FORMAT_EXAMPLE": ''' +{ + "Required Python third-party packages": [ + "flask==1.1.2", + "bcrypt==3.2.0" + ], + "Required Other language third-party packages": [ + "No third-party ..." + ], + "Full API spec": """ + openapi: 3.0.0 + ... + description: A JSON object ... + """, + "Logic Analysis": [ + ["game.py","Contains..."] + ], + "Task list": [ + "game.py" + ], + "Shared Knowledge": """ + 'game.py' contains ... + """, + "Difference Description": """ + The ... + """, + "Anything UNCLEAR": "We need ... how to start." +} +''', + }, + "markdown": { + "PROMPT_TEMPLATE": """ +# Context +{context} + +## Legacy Design +{legacy} + +## Format example +{format_example} +----- +Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. + +## Required Python third-party packages: Provided in requirements.txt format + +## Required Other language third-party packages: Provided in requirements.txt format + +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. + +## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first + +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first + +## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. + +## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. + +## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. + +""", + "FORMAT_EXAMPLE": ''' +--- +## Required Python third-party packages +```python +""" +flask==1.1.2 +bcrypt==3.2.0 +""" +``` + +## Required Other language third-party packages +```python +""" +No third-party ... +""" +``` + +## Full API spec +```python +""" +openapi: 3.0.0 +... +description: A JSON object ... +""" +``` + +## Logic Analysis +```python +[ + ["game.py", "Contains ..."], +] +``` + +## Task list +```python +[ + "game.py", +] +``` + +## Shared Knowledge +```python +""" +'game.py' contains ... +""" +``` + +## Difference Description +```python +""" +The ... +""" +``` + +## Anything UNCLEAR +We need ... how to start. +--- +''', + }, +} +OUTPUT_MAPPING = { + "Required Python third-party packages": (List[str], ...), + "Required Other language third-party packages": (List[str], ...), + "Full API spec": (str, ...), + "Logic Analysis": (List[List[str]], ...), + "Task list": (List[str], ...), + "Shared Knowledge": (str, ...), + "Difference Description": (str, ...), + "Anything UNCLEAR": (str, ...), +} + + +class RefineTasks(Action): + def __init__(self, name="CreateTasks", context=None, llm=None): + super().__init__(name, context, llm) + + def _save(self, context, rsp): + if context[-1].instruct_content: + ws_name = context[-1].instruct_content.dict()["Python package name"] + else: + ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) + file_path = WORKSPACE_ROOT / ws_name / "docs/api_spec_and_tasks.md" + file_path.write_text(json_to_markdown(rsp.instruct_content.dict())) + + # Write requirements.txt + requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" + requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) + + async def run(self, context, legacy, format=CONFIG.prompt_format): + prompt_template, format_example = get_template(templates, format) + prompt = prompt_template.format(context=context, legacy=legacy, format_example=format_example) + rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) + self._save(context, rsp) + return rsp + + +class AssignTasks(Action): + async def run(self, *args, **kwargs): + # Here you should implement the actual action + pass diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index c000805c5..663ae0929 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -7,6 +7,7 @@ """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action +from metagpt.actions.refine_design_api import RefineDesign from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message @@ -55,7 +56,14 @@ class WriteCode(Action): if self._is_invalid(filename): return - design = [i for i in context if i.cause_by == WriteDesign][0] + # FIXME: 需要适配increment + # design = [i for i in context if i.cause_by == WriteDesign][0] + design = [] + for i in context: + if i.cause_by == WriteDesign: + design.append(i) + elif i.cause_by == RefineDesign: + design.append(i) ws_name = CodeParser.parse_str(block="Python package name", text=design.content) ws_path = WORKSPACE_ROOT / ws_name diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 15d5fe5b1..dddcd2a8b 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -6,9 +6,13 @@ @File : architect.py """ -from metagpt.actions import WritePRD +from metagpt.actions import WritePRD, ActionOutput from metagpt.actions.design_api import WriteDesign +from metagpt.actions.refine_design_api import RefineDesign +from metagpt.actions.refine_prd import RefinePRD +from metagpt.logs import logger from metagpt.roles import Role +from metagpt.schema import Message class Architect(Role): @@ -23,17 +27,43 @@ class Architect(Role): """ def __init__( - self, - name: str = "Bob", - profile: str = "Architect", - goal: str = "Design a concise, usable, complete python system", - constraints: str = "Try to specify good open source tools as much as possible", + self, + name: str = "Bob", + profile: str = "Architect", + goal: str = "Design a concise, usable, complete python system", + constraints: str = "Try to specify good open source tools as much as possible", + legacy: str = "", + increment: bool = False, ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) + self.legacy = legacy + self.increment = increment # Initialize actions specific to the Architect role - self._init_actions([WriteDesign]) - # Set events or actions the Architect should watch or be aware of - self._watch({WritePRD}) + if self.increment: + self._init_actions([RefineDesign]) + self._watch({RefinePRD}) + else: + self._init_actions([WriteDesign]) + self._watch({WritePRD}) + + async def _act(self) -> Message: + if self.increment: + logger.info(f"{self._setting}: ready to RefineDesign") + response = await self._rc.todo.run(self._rc.history, self.legacy) + + else: + logger.info(f"{self._setting}: ready to WriteDesign") + response = await self._rc.todo.run(self._rc.history) + + if isinstance(response, ActionOutput): + msg = Message(content=response.content, instruct_content=response.instruct_content, + role=self.profile, cause_by=type(self._rc.todo)) + else: + msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + logger.debug(f"{response}") + + return msg diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 1f6685b38..0a0107636 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -11,6 +11,8 @@ from collections import OrderedDict from pathlib import Path from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks +from metagpt.actions.refine_design_api import RefineDesign +from metagpt.actions.refine_project_management import RefineTasks from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.roles import Role @@ -68,14 +70,21 @@ class Engineer(Role): constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", n_borg: int = 1, use_code_review: bool = False, + legacy: str = "", + increment: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review - if self.use_code_review: + self.legacy = legacy + self.increment = increment + if self.use_code_review or self.increment: self._init_actions([WriteCode, WriteCodeReview]) - self._watch([WriteTasks]) + if self.increment: + self._watch([RefineTasks]) + else: + self._watch([WriteTasks]) self.todos = [] self.n_borg = n_borg @@ -96,7 +105,10 @@ class Engineer(Role): return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] + if self.increment: + msg = self._rc.memory.get_by_action(RefineDesign)[-1] + else: + msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) @@ -167,6 +179,55 @@ class Engineer(Role): ) return msg + async def _act_increment(self, legacy) -> Message: + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + flag = True + for todo in self.todos: + """ + # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): + 1. All from Architect + 2. All from ProjectManager + 3. Do we need other codes (currently needed)? + TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. + """ + context = [] + + if self.increment: + msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks, WriteCode]) + else: + msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) + + for m in msg: + context.append(m.content) + context_str = "\n".join(context) + # Refine code or Write code + if flag and self.increment: + code = legacy + flag = False + else: + code = await WriteCode().run(context=context_str, filename=todo) + + # Code review + if self.use_code_review: + try: + rewrite_code = await WriteCode().run(context=context_str, code=code, filename=todo) + code = rewrite_code + except Exception as e: + logger.error("code review failed!", e) + pass + file_path = self.write_file(todo, code) + msg = Message(content=code, role=self.profile, cause_by=WriteCode) + self._rc.memory.add(msg) + + code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg_all.append(code_msg) + + logger.info(f"Done {self.get_workspace()} generating.") + msg = Message( + content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" + ) + return msg + async def _act_sp_precision(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: @@ -207,7 +268,12 @@ class Engineer(Role): async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" - logger.info(f"{self._setting}: ready to WriteCode") + if self.increment: + logger.info(f"{self._setting}: ready to RefineWriteCode") + else: + logger.info(f"{self._setting}: ready to WriteCode") if self.use_code_review: return await self._act_sp_precision() + elif self.increment: + return await self._act_increment(self.legacy) return await self._act_sp() diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index a58ea5385..0b2d83ed0 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -5,8 +5,11 @@ @Author : alexanderwu @File : product_manager.py """ -from metagpt.actions import BossRequirement, WritePRD +from metagpt.actions import BossRequirement, WritePRD, ActionOutput +from metagpt.actions.refine_prd import RefinePRD +from metagpt.logs import logger from metagpt.roles import Role +from metagpt.schema import Message class ProductManager(Role): @@ -26,6 +29,9 @@ class ProductManager(Role): profile: str = "Product Manager", goal: str = "Efficiently create a successful product", constraints: str = "", + difference_description: str = "", + legacy: str = "", + increment: bool = False, ) -> None: """ Initializes the ProductManager role with given attributes. @@ -37,5 +43,31 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([WritePRD]) + self.difference_description = difference_description + self.legacy = legacy + self.increment = increment + + if self.increment: + self._init_actions([RefinePRD]) + else: + self._init_actions([WritePRD]) self._watch([BossRequirement]) + + async def _act(self) -> Message: + if self.increment: + logger.info(f"{self._setting}: ready to RefinePRD") + response = await self._rc.todo.run(self._rc.history, self.difference_description, self.legacy) + + else: + logger.info(f"{self._setting}: ready to WritePRD") + response = await self._rc.todo.run(self._rc.history) + + if isinstance(response, ActionOutput): + msg = Message(content=response.content, instruct_content=response.instruct_content, + role=self.profile, cause_by=type(self._rc.todo)) + else: + msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + logger.debug(f"{response}") + + return msg diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 7e7c5699d..21a196d21 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -5,9 +5,13 @@ @Author : alexanderwu @File : project_manager.py """ -from metagpt.actions import WriteTasks +from metagpt.actions import WriteTasks, ActionOutput from metagpt.actions.design_api import WriteDesign +from metagpt.actions.refine_design_api import RefineDesign +from metagpt.actions.refine_project_management import RefineTasks +from metagpt.logs import logger from metagpt.roles import Role +from metagpt.schema import Message class ProjectManager(Role): @@ -27,6 +31,8 @@ class ProjectManager(Role): profile: str = "Project Manager", goal: str = "Improve team efficiency and deliver with quality and quantity", constraints: str = "", + increment: bool = False, + legacy: str = "", ) -> None: """ Initializes the ProjectManager role with given attributes. @@ -38,5 +44,31 @@ class ProjectManager(Role): constraints (str): Constraints or limitations for the project manager. """ super().__init__(name, profile, goal, constraints) - self._init_actions([WriteTasks]) - self._watch([WriteDesign]) + self.increment = increment + self.legacy = legacy + + if self.increment: + self._init_actions([RefineTasks]) + self._watch([RefineDesign]) + else: + self._init_actions([WriteTasks]) + self._watch([WriteDesign]) + + async def _act(self) -> Message: + if self.increment: + logger.info(f"{self._setting}: ready to RefineTasks") + response = await self._rc.todo.run(self._rc.history, self.legacy) + + else: + logger.info(f"{self._setting}: ready to WriteTasks") + response = await self._rc.todo.run(self._rc.history) + + if isinstance(response, ActionOutput): + msg = Message(content=response.content, instruct_content=response.instruct_content, + role=self.profile, cause_by=type(self._rc.todo)) + else: + msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) + self._rc.memory.add(msg) + logger.debug(f"{response}") + + return msg diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 0251176f7..a39ca1001 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -205,6 +205,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") + response = await self._rc.todo.run(self._rc.important_memory) # logger.info(response) if isinstance(response, ActionOutput): diff --git a/startup.py b/startup.py index e9fbf94d3..7e8e0a692 100644 --- a/startup.py +++ b/startup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- import asyncio +import os import fire @@ -16,26 +17,57 @@ from metagpt.team import Team async def startup( idea: str, + difference_description: str = "", + path: str = "", investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False, implement: bool = True, + increment: bool = False, ): """Run a startup. Be a boss.""" company = Team() - company.hire( - [ - ProductManager(), - Architect(), - ProjectManager(), - ] - ) + + if increment: + # 读取文件 + prd_path = os.path.join(path, 'docs/prd.md') + design_path = os.path.join(path, 'docs/system_design.md') + api_spec_path = os.path.join(path, 'docs/api_spec_and_tasks.md') + code_path = os.path.join(path, os.path.basename(path)) + + with open(prd_path, 'r', encoding='utf-8') as f: + legacy_prd = f.read() + + with open(design_path, 'r', encoding='utf-8') as f: + legacy_design = f.read() + + with open(api_spec_path, 'r', encoding='utf-8') as f: + legacy_api_spec = f.read() + + # 遍历文件夹，获取所有代码文件 + legacy_code = '' + for root, dirs, files in os.walk(code_path): + filenames = [filename for filename in files if filename.endswith('.py')] + legacy_code += f'There are {len(files)} scripts in the current folder: {", ".join(filenames)}\n\n' + for file in files: + if file.endswith('.py'): + with open(os.path.join(root, file), 'r', encoding='utf-8') as f: + legacy_code += f.read() + '\n\n' + + company.hire( + [ProductManager(difference_description=difference_description, legacy=legacy_prd, increment=increment), + Architect(legacy=legacy_design, increment=increment), + ProjectManager(legacy=legacy_api_spec, increment=increment)]) + else: + company.hire([ProductManager(), Architect(), ProjectManager()]) # if implement or code_review - if implement or code_review: + if (implement or code_review) and not increment: # developing features: implement the idea company.hire([Engineer(n_borg=5, use_code_review=code_review)]) + elif (implement or code_review) and increment: + company.hire([Engineer(n_borg=5, use_code_review=code_review, legacy=legacy_code, increment=increment)]) if run_tests: # developing features: run tests on the spot and identify bugs @@ -49,11 +81,14 @@ async def startup( def main( idea: str, + difference_description: str = "", + path: str = "", investment: float = 3.0, n_round: int = 5, code_review: bool = True, run_tests: bool = False, implement: bool = True, + increment: bool = False, ): """ We are a software startup comprised of AI. By investing in us, @@ -65,7 +100,8 @@ def main( :param code_review: Whether to use code review. :return: """ - asyncio.run(startup(idea, investment, n_round, code_review, run_tests, implement)) + asyncio.run( + startup(idea, difference_description, path, investment, n_round, code_review, run_tests, implement, increment)) if __name__ == "__main__": From 1ea3c0c9f3149d1ccd4cd36ea8a2bd738c683a52 Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Tue, 21 Nov 2023 14:42:45 +0800 Subject: [PATCH 2/6] add increment development function --- metagpt/roles/role.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index a39ca1001..0251176f7 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -205,7 +205,6 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - response = await self._rc.todo.run(self._rc.important_memory) # logger.info(response) if isinstance(response, ActionOutput): From 8ff833f1e355d5adb5341dbbd8e3fd4265c110f3 Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Fri, 24 Nov 2023 18:22:06 +0800 Subject: [PATCH 3/6] update increment development, add bug fix function --- metagpt/actions/refine_design_api.py | 25 ++- metagpt/actions/refine_prd.py | 17 +- metagpt/actions/refine_project_management.py | 34 ++-- metagpt/actions/write_code_refine.py | 80 ++++++++++ metagpt/roles/engineer.py | 154 +++++++++++++++---- metagpt/roles/qa_engineer.py | 36 ++++- startup.py | 68 +++++--- 7 files changed, 334 insertions(+), 80 deletions(-) create mode 100644 metagpt/actions/write_code_refine.py diff --git a/metagpt/actions/refine_design_api.py b/metagpt/actions/refine_design_api.py index 591db175a..f1c231525 100644 --- a/metagpt/actions/refine_design_api.py +++ b/metagpt/actions/refine_design_api.py @@ -1,5 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +import re import shutil from pathlib import Path from typing import List @@ -25,7 +26,7 @@ templates = { ## Format example {format_example} ----- -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Role: You are an architect; the goal is to perform incremental development and design a state-of-the-art (SOTA) PEP8-compliant Python system based on the context and the provided difference descriptions. Make the best use of good open source tools. Requirement: Fill in the following missing information based on the context, each section name is a key in json Max Output: 8192 chars or 2048 tokens. Try to use them up. @@ -83,7 +84,7 @@ and only output the json inside this tag, nothing else ## Format example {format_example} ----- -Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools +Role: You are an architect; the goal is to perform incremental development and design a state-of-the-art (SOTA) PEP8-compliant Python system based on the context and the provided difference descriptions. Make the best use of good open source tools. Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately Max Output: 8192 chars or 2048 tokens. Try to use them up. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -179,6 +180,25 @@ class RefineDesign(Action): pass # Folder does not exist, but we don't care workspace.mkdir(parents=True, exist_ok=True) + def create_or_increment_workspace(self, workspace: Path): + # 如果工作空间已存在，添加数字以区分 + original_workspace = workspace + index = 1 + while workspace.exists(): + ws_name_match = re.match(r'^(.*)_([\d]+)$', original_workspace.name) + if ws_name_match: + base_name, existing_index = ws_name_match.groups() + index = int(existing_index) + index += 1 + workspace = original_workspace.parent / f"{base_name}_{index}" + else: + workspace = original_workspace.parent / f"{original_workspace.name}_{index}" + index += 1 + + # 创建工作空间，包括所有必要的父文件夹 + workspace.mkdir(parents=True, exist_ok=True) + return workspace + async def _save_prd(self, docs_path, resources_path, context): prd_file = docs_path / "prd.md" if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: @@ -208,6 +228,7 @@ class RefineDesign(Action): else: ws_name = CodeParser.parse_str(block="Python package name", text=system_design) workspace = WORKSPACE_ROOT / ws_name + # workspace = self.create_or_increment_workspace(workspace) self.recreate_workspace(workspace) docs_path = workspace / "docs" resources_path = workspace / "resources" diff --git a/metagpt/actions/refine_prd.py b/metagpt/actions/refine_prd.py index 1e2709c6b..81cb02af8 100644 --- a/metagpt/actions/refine_prd.py +++ b/metagpt/actions/refine_prd.py @@ -43,13 +43,15 @@ quadrantChart ## Format example {format_example} ----- -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product based on the new requirements, the difference description and Legacy PRD. +Role: You are a professional Product Manager tasked with overseeing incremental development and crafting Product Requirements Documents (PRDs) for a concise, usable, and efficient product. Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design ## New Requirements: Provide as Plain text and place the new requirements here ## Difference Description: Provide as Python list[str], up to 5 clear, difference descriptions. If the requirement itself is simple, the difference description should also be simple +## Incremental Development Plan: Provide as Python list[str], up to 5 clear, incremental development plans. If the requirement itself is simple, the incremental development plan should also be simple + ## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple ## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less @@ -72,9 +74,10 @@ and only output the json inside this tag, nothing else [CONTENT] { "New Requirements": "", - "Difference Description": "", + "Difference Description": [], "Search Information": "", "Requirements": "", + "Incremental Development Plan": [], "Product Goals": [], "User Stories": [], "Competitive Analysis": [], @@ -138,7 +141,7 @@ quadrantChart ## Format example {format_example} ----- -Role: You are a professional product manager; the goal is to design a concise, usable, efficient product +Role: You are a professional Product Manager tasked with overseeing incremental development and crafting Product Requirements Documents (PRDs) for a concise, usable, and efficient product. Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. @@ -146,6 +149,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD W ## Difference Description: Provide as Python list[str], up to 5 clear, difference descriptions. If the requirement itself is simple, the difference description should also be simple +## Incremental Development Plan: Provide as Python list[str], up to 5 clear, incremental development plans. If the requirement itself is simple, the incremental development plan should also be simple + ## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple ## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less @@ -172,6 +177,11 @@ The boss ... "...", ] +## Incremental Development Plan +[ + "It ...", +] + ## Product Goals ```python [ @@ -225,6 +235,7 @@ INCREMENT_OUTPUT_MAPPING = { "New Requirements": (str, ...), # "Major Enhancements": (List[str], ...), "Difference Description": (List[str], ...), + "Incremental Development Plan": (List[str], ...), "Product Goals": (List[str], ...), "User Stories": (List[str], ...), "Competitive Analysis": (List[str], ...), diff --git a/metagpt/actions/refine_project_management.py b/metagpt/actions/refine_project_management.py index 354996f3e..7d9e118d8 100644 --- a/metagpt/actions/refine_project_management.py +++ b/metagpt/actions/refine_project_management.py @@ -21,7 +21,7 @@ templates = { ## Format example {format_example} ----- -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Role: You are a project manager; the goal is to perform incremental development based on the context and difference descriptions and the legacy. Break down tasks according to PRD/technical design, provide a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -31,14 +31,14 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. +## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. + ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. - ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, @@ -58,6 +58,9 @@ and only output the json inside this tag, nothing else ... description: A JSON object ... """, + "Difference Description": """ + The ... + """, "Logic Analysis": [ ["game.py","Contains..."] ], @@ -67,9 +70,6 @@ and only output the json inside this tag, nothing else "Shared Knowledge": """ 'game.py' contains ... """, - "Difference Description": """ - The ... - """, "Anything UNCLEAR": "We need ... how to start." } ''', @@ -85,7 +85,7 @@ and only output the json inside this tag, nothing else ## Format example {format_example} ----- -Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules +Role: You are a project manager; the goal is to perform incremental development based on the context and difference descriptions and the legacy. Break down tasks according to PRD/technical design, provide a task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. @@ -95,14 +95,14 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. +## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. + ## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first ## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first ## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. -## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. - ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. """, @@ -132,6 +132,13 @@ description: A JSON object ... """ ``` +## Difference Description +```python +""" +The ... +""" +``` + ## Logic Analysis ```python [ @@ -153,13 +160,6 @@ description: A JSON object ... """ ``` -## Difference Description -```python -""" -The ... -""" -``` - ## Anything UNCLEAR We need ... how to start. --- @@ -170,10 +170,10 @@ OUTPUT_MAPPING = { "Required Python third-party packages": (List[str], ...), "Required Other language third-party packages": (List[str], ...), "Full API spec": (str, ...), + "Difference Description": (str, ...), "Logic Analysis": (List[List[str]], ...), "Task list": (List[str], ...), "Shared Knowledge": (str, ...), - "Difference Description": (str, ...), "Anything UNCLEAR": (str, ...), } diff --git a/metagpt/actions/write_code_refine.py b/metagpt/actions/write_code_refine.py new file mode 100644 index 000000000..527952aed --- /dev/null +++ b/metagpt/actions/write_code_refine.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from metagpt.actions.action import Action +from metagpt.logs import logger +from metagpt.schema import Message +from metagpt.utils.common import CodeParser +from tenacity import retry, stop_after_attempt, wait_fixed + + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to conduct incremental development, which includes reviewing existing code, providing modification suggestions, rewriting code, and optimizing the codebase. Existing code and logic that need to be retained must also appear in the code after incremental development, do not omit it. Ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +## Code Review: Based on the following context and legacy code, and following the checklist, provide key, clear, concise, and specific code modification suggestions, up to 5. +``` +1. Check 0: Is the code implemented as per the requirements? +2. Check 1: Are there any issues with the code logic? +3. Check 2: Does the existing code follow the "Data structures and interface definitions"? +4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? +5. Check 4: Does the code have unnecessary or lack dependencies? +``` + +## Incremental Development: {filename} Based on the findings from the "Code Review," context, and the legacy code, conduct incremental development by rewriting, optimizing, and adding new code using triple quotes. +----- +# Context +{context} + +## Legacy Code +You are tasked with conducting incremental development in the existing code and creating a new code file, {filename}, based on the provided legacy code and above information. +``` +{code} +``` +----- + +## Format example +----- +{format_example} +----- + +""" + +FORMAT_EXAMPLE = """ + +## Code Review +1. The code ... +2. ... +3. ... +4. ... +5. ... + +## Incremental Development: {filename} +```python +## {filename} - Incremental Development +... +``` +""" + + + +class WriteCodeRefine(Action): + def __init__(self, name="WriteCodeRefine", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) + async def write_code(self, prompt): + code_rsp = await self._aask(prompt) + code = CodeParser.parse_code(block="", text=code_rsp) + return code + + async def run(self, context, code, filename): + format_example = FORMAT_EXAMPLE.format(filename=filename) + prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) + logger.info(f'Code refine {filename}..') + code = await self.write_code(prompt) + # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) + # self._save(context, filename, code) + return code + \ No newline at end of file diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 0a0107636..13034acaa 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -6,13 +6,16 @@ @File : engineer.py """ import asyncio +import re import shutil from collections import OrderedDict from pathlib import Path +from typing import List -from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks +from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks, BossRequirement from metagpt.actions.refine_design_api import RefineDesign from metagpt.actions.refine_project_management import RefineTasks +from metagpt.actions.write_code_refine import WriteCodeRefine from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.roles import Role @@ -72,6 +75,8 @@ class Engineer(Role): use_code_review: bool = False, legacy: str = "", increment: bool = False, + bug_msgs: List = None, + bug_fix: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) @@ -79,13 +84,20 @@ class Engineer(Role): self.use_code_review = use_code_review self.legacy = legacy self.increment = increment + self.bug_msgs = bug_msgs + self.bug_fix = bug_fix if self.use_code_review or self.increment: - self._init_actions([WriteCode, WriteCodeReview]) + self._init_actions([WriteCode, WriteCodeReview, WriteCodeRefine]) + if self.increment: self._watch([RefineTasks]) + self.todos = [] + elif self.bug_fix: + self._watch([BossRequirement]) + self.todos = [] else: self._watch([WriteTasks]) - self.todos = [] + self.todos = [] self.n_borg = n_borg @classmethod @@ -94,6 +106,18 @@ class Engineer(Role): return task_msg.instruct_content.dict().get("Task list") return CodeParser.parse_file_list(block="Task list", text=task_msg.content) + @classmethod + def parse_todos(self, bug_context: List) -> list[str]: + for msg in bug_context: + if msg.sent_from == "ProjectManager": + content = msg.content + tasks_section = re.search(r"## Task list\n\n(.*?)(\n\n|$)", content, re.DOTALL) + if tasks_section: + tasks = tasks_section.group(1).split("\n") + tasks = [task.strip("-").strip() for task in tasks] + return tasks + return [] + @classmethod def parse_code(self, code_text: str) -> str: return CodeParser.parse_code(block="", text=code_text) @@ -107,13 +131,17 @@ class Engineer(Role): def get_workspace(self) -> Path: if self.increment: msg = self._rc.memory.get_by_action(RefineDesign)[-1] + elif self.bug_fix: + msg = self._rc.memory.get_by_action(BossRequirement)[-1] else: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) + workspace = workspace if workspace else "src" # Codes are written in workspace/{package_name}/{package_name} return WORKSPACE_ROOT / workspace / workspace + # return self.create_or_increment_workspace(WORKSPACE_ROOT, workspace) def recreate_workspace(self): workspace = self.get_workspace() @@ -123,8 +151,7 @@ class Engineer(Role): pass # The folder does not exist, but we don't care workspace.mkdir(parents=True, exist_ok=True) - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() + def write_file(self, workspace: Path, filename: str, code: str) -> Path: filename = filename.replace('"', "").replace("\n", "") file = workspace / filename file.parent.mkdir(parents=True, exist_ok=True) @@ -134,7 +161,10 @@ class Engineer(Role): def recv(self, message: Message) -> None: self._rc.memory.add(message) if message in self._rc.important_memory: - self.todos = self.parse_tasks(message) + if not self.bug_fix: + self.todos = self.parse_tasks(message) + else: + self.todos = self.parse_todos(self.bug_msgs) async def _act_mp(self) -> Message: # self.recreate_workspace() @@ -161,12 +191,13 @@ class Engineer(Role): async def _act_sp(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + workspace = self.get_workspace() for todo in self.todos: code = await WriteCode().run(context=self._rc.history, filename=todo) # logger.info(todo) # logger.info(code_rsp) # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) + file_path = self.write_file(workspace, todo, code) msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) @@ -181,41 +212,76 @@ class Engineer(Role): async def _act_increment(self, legacy) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + workspace = self.get_workspace() flag = True + # legacy_codes = legacy.split('---') for todo in self.todos: - """ - # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): - 1. All from Architect - 2. All from ProjectManager - 3. Do we need other codes (currently needed)? - TODO: The goal is not to need it. After clear task decomposition, based on the design idea, you should be able to write a single file without needing other codes. If you can't, it means you need a clearer definition. This is the key to writing longer code. - """ context = [] - if self.increment: - msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks, WriteCode]) - else: - msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode]) - + msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks, WriteCodeRefine]) for m in msg: context.append(m.content) context_str = "\n".join(context) + code = legacy + # Refine code or Write code - if flag and self.increment: - code = legacy - flag = False - else: - code = await WriteCode().run(context=context_str, filename=todo) + # if self.increment and len(legacy_codes) > 0: + # code = legacy_codes.pop(0) # Code review - if self.use_code_review: - try: - rewrite_code = await WriteCode().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - file_path = self.write_file(todo, code) + try: + rewrite_code = await WriteCodeRefine().run(context=context_str, code=code, filename=todo) + code = rewrite_code + except Exception as e: + logger.error("code review failed!", e) + pass + + # code = await WriteCode().run(context=context_str, filename=todo) + + file_path = self.write_file(workspace, todo, code) + msg = Message(content=code, role=self.profile, cause_by=WriteCode) + self._rc.memory.add(msg) + + code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg_all.append(code_msg) + + logger.info(f"Done {self.get_workspace()} generating.") + msg = Message( + content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" + ) + return msg + + async def _act_bug_fix(self, bug_msgs) -> Message: + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + workspace = self.get_workspace() + flag = True + # legacy_codes = legacy.split('---') + for todo in self.todos: + context = [] + + for m in bug_msgs: + if m.sent_from != "Engineer": + context.append(m.content) + context.append(m.content) + context_str = "\n".join(context) + code = [m.content for m in bug_msgs if m.sent_from == "Engineer"] + code = "\n".join(code) + + # Refine code or Write code + # if self.increment and len(legacy_codes) > 0: + # code = legacy_codes.pop(0) + + # Code review + try: + rewrite_code = await WriteCodeRefine().run(context=context_str, code=code, filename=todo) + code = rewrite_code + except Exception as e: + logger.error("code review failed!", e) + pass + + # code = await WriteCode().run(context=context_str, filename=todo) + + file_path = self.write_file(workspace, todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -230,6 +296,7 @@ class Engineer(Role): async def _act_sp_precision(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + workspace = self.get_workspace() for todo in self.todos: """ # Select essential information from the historical data to reduce the length of the prompt (summarized from human experience): @@ -253,7 +320,7 @@ class Engineer(Role): except Exception as e: logger.error("code review failed!", e) pass - file_path = self.write_file(todo, code) + file_path = self.write_file(workspace, todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) @@ -266,14 +333,35 @@ class Engineer(Role): ) return msg + async def _observe(self) -> int: + if self.bug_fix: + msg = Message( + content=self.bug_msgs[0].content + "\n---\n" + self.legacy, + role=self.profile, + cause_by=BossRequirement, + sent_from=self.profile, + send_to=self.profile, + ) + self._publish_message(msg) + await super()._observe() + self._rc.news = [ + msg for msg in self._rc.news if msg.send_to == self.profile + ] # only relevant msgs count as observed news + return len(self._rc.news) + async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" if self.increment: logger.info(f"{self._setting}: ready to RefineWriteCode") + elif self.bug_fix: + logger.info(f"{self._setting}: ready to BugFix") else: logger.info(f"{self._setting}: ready to WriteCode") + if self.use_code_review: return await self._act_sp_precision() elif self.increment: return await self._act_increment(self.legacy) + elif self.bug_fix: + return await self._act_bug_fix(self.bug_msgs) return await self._act_sp() diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index a763c2ce8..c67d42b5c 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -14,8 +14,9 @@ from metagpt.actions import ( WriteCode, WriteCodeReview, WriteDesign, - WriteTest, + WriteTest, BossRequirement, ) +from metagpt.actions.write_code_refine import WriteCodeRefine from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.roles import Role @@ -32,12 +33,20 @@ class QaEngineer(Role): goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", test_round_allowed=5, + legacy="", + bug_context="", ): super().__init__(name, profile, goal, constraints) - self._init_actions( - [WriteTest] - ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates - self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) + self.legacy = legacy + self.bug_context = bug_context + if self.bug_context: + self._init_actions([WriteTest]) + self._watch([WriteCode, WriteCodeRefine, WriteTest, RunCode, DebugError]) + else: + self._init_actions( + [WriteTest] + ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates + self._watch([WriteCode, WriteCodeReview, WriteTest, RunCode, DebugError]) self.test_round = 0 self.test_round_allowed = test_round_allowed @@ -48,10 +57,14 @@ class QaEngineer(Role): return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self, return_proj_dir=True) -> Path: - msg = self._rc.memory.get_by_action(WriteDesign)[-1] + if self.bug_context: + msg = self._rc.memory.get_by_action(WriteCodeRefine)[-1] + else: + msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: return WORKSPACE_ROOT / "src" workspace = self.parse_workspace(msg) + workspace = workspace if workspace else "src" # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: return WORKSPACE_ROOT / workspace @@ -146,6 +159,15 @@ class QaEngineer(Role): self._publish_message(msg) async def _observe(self) -> int: + if self.bug_context: + msg = Message( + content=self.bug_context + "\n---\n" + self.legacy, + role=self.profile, + cause_by=BossRequirement, + sent_from=self.profile, + send_to=self.profile, + ) + self._publish_message(msg) await super()._observe() self._rc.news = [ msg for msg in self._rc.news if msg.send_to == self.profile @@ -166,7 +188,7 @@ class QaEngineer(Role): for msg in self._rc.news: # Decide what to do based on observed msg type, currently defined by human, # might potentially be moved to _think, that is, let the agent decides for itself - if msg.cause_by in [WriteCode, WriteCodeReview]: + if msg.cause_by in [WriteCode, WriteCodeReview, WriteCodeRefine]: # engineer wrote a code, time to write a test for it await self._write_test(msg) elif msg.cause_by in [WriteTest, DebugError]: diff --git a/startup.py b/startup.py index 7e8e0a692..8bb8517c9 100644 --- a/startup.py +++ b/startup.py @@ -2,6 +2,7 @@ # -*- coding: utf-8 -*- import asyncio import os +import platform import fire @@ -12,29 +13,32 @@ from metagpt.roles import ( ProjectManager, QaEngineer, ) +from metagpt.schema import Message from metagpt.team import Team +from metagpt.utils.special_tokens import MSG_SEP async def startup( idea: str, difference_description: str = "", - path: str = "", + project_path: str = "", investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False, implement: bool = True, increment: bool = False, + bug_fix: bool = False, ): """Run a startup. Be a boss.""" company = Team() - if increment: + if increment or bug_fix: # 读取文件 - prd_path = os.path.join(path, 'docs/prd.md') - design_path = os.path.join(path, 'docs/system_design.md') - api_spec_path = os.path.join(path, 'docs/api_spec_and_tasks.md') - code_path = os.path.join(path, os.path.basename(path)) + prd_path = os.path.join(project_path, 'docs/prd.md') + design_path = os.path.join(project_path, 'docs/system_design.md') + api_spec_and_tasks_path = os.path.join(project_path, 'docs/api_spec_and_tasks.md') + code_path = os.path.join(project_path, os.path.basename(project_path)) with open(prd_path, 'r', encoding='utf-8') as f: legacy_prd = f.read() @@ -42,34 +46,59 @@ async def startup( with open(design_path, 'r', encoding='utf-8') as f: legacy_design = f.read() - with open(api_spec_path, 'r', encoding='utf-8') as f: - legacy_api_spec = f.read() + with open(api_spec_and_tasks_path, 'r', encoding='utf-8') as f: + legacy_api_spec_and_tasks = f.read() # 遍历文件夹，获取所有代码文件 legacy_code = '' for root, dirs, files in os.walk(code_path): filenames = [filename for filename in files if filename.endswith('.py')] - legacy_code += f'There are {len(files)} scripts in the current folder: {", ".join(filenames)}\n\n' + legacy_code += f'There are {len(files)} scripts in the current folder: {", ".join(filenames)}\n---\n' for file in files: if file.endswith('.py'): with open(os.path.join(root, file), 'r', encoding='utf-8') as f: - legacy_code += f.read() + '\n\n' + legacy_code += f.read() + '\n---\n' - company.hire( - [ProductManager(difference_description=difference_description, legacy=legacy_prd, increment=increment), - Architect(legacy=legacy_design, increment=increment), - ProjectManager(legacy=legacy_api_spec, increment=increment)]) + if bug_fix: + boss_msg = Message( + content=f"Boss's requirement\n:{idea}\n---\nBoss's difference description:{difference_description}\n---\n", + sent_from="Boss", + ) + product_manager_msg = Message( + content=f"Product Manager's prd legacy:\n{legacy_prd}\n---\n", + sent_from="ProductManager" + ) + architect_msg = Message( + content=f"Architect's design legacy:\n{legacy_design}\n---\n", + sent_from="Architect" + ) + project_manager_msg = Message( + content=f"Project Manager's api spec and tasks legacy:\n{legacy_api_spec_and_tasks}\n---\n", + sent_from="ProjectManager" + ) + engineer_msg = Message( + content=f"Engineer's code legacy:\n{legacy_code}\n---\n", + sent_from="Engineer" + ) + bug_msgs = [boss_msg, product_manager_msg, architect_msg, project_manager_msg, engineer_msg] + else: + company.hire( + [ProductManager(difference_description=difference_description, legacy=legacy_prd, increment=increment), + Architect(legacy=legacy_design, increment=increment), + ProjectManager(legacy=legacy_api_spec_and_tasks, increment=increment)]) else: company.hire([ProductManager(), Architect(), ProjectManager()]) # if implement or code_review - if (implement or code_review) and not increment: + if bug_fix: + company.hire([Engineer(n_borg=5, bug_msgs=bug_msgs, bug_fix=bug_fix)]) + elif implement or code_review: # developing features: implement the idea company.hire([Engineer(n_borg=5, use_code_review=code_review)]) elif (implement or code_review) and increment: company.hire([Engineer(n_borg=5, use_code_review=code_review, legacy=legacy_code, increment=increment)]) - if run_tests: + if run_tests or bug_fix: # developing features: run tests on the spot and identify bugs # (bug fixing capability comes soon!) company.hire([QaEngineer()]) @@ -82,13 +111,14 @@ async def startup( def main( idea: str, difference_description: str = "", - path: str = "", + project_path: str = "", investment: float = 3.0, n_round: int = 5, code_review: bool = True, run_tests: bool = False, implement: bool = True, increment: bool = False, + bug_fix: bool = False, ): """ We are a software startup comprised of AI. By investing in us, @@ -100,8 +130,10 @@ def main( :param code_review: Whether to use code review. :return: """ + + asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run( - startup(idea, difference_description, path, investment, n_round, code_review, run_tests, implement, increment)) + startup(idea, difference_description, project_path, investment, n_round, code_review, run_tests, implement, increment, bug_fix)) if __name__ == "__main__": From a50a1f738fba5992e3ef3cfbe917a81a6675576d Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Wed, 29 Nov 2023 17:31:15 +0800 Subject: [PATCH 4/6] update increment development, add write_code_guide.py file --- metagpt/actions/refine_design_api.py | 73 +++---- metagpt/actions/refine_prd.py | 192 ++----------------- metagpt/actions/refine_project_management.py | 118 +++++------- metagpt/actions/write_code_guide.py | 67 +++++++ metagpt/actions/write_code_refine.py | 61 +++--- metagpt/environment.py | 13 ++ metagpt/roles/architect.py | 5 +- metagpt/roles/engineer.py | 122 ++++++------ metagpt/roles/product_manager.py | 7 +- metagpt/roles/project_manager.py | 9 +- metagpt/team.py | 6 + startup.py | 66 +++---- 12 files changed, 290 insertions(+), 449 deletions(-) create mode 100644 metagpt/actions/write_code_guide.py diff --git a/metagpt/actions/refine_design_api.py b/metagpt/actions/refine_design_api.py index f1c231525..909fa6db9 100644 --- a/metagpt/actions/refine_design_api.py +++ b/metagpt/actions/refine_design_api.py @@ -3,7 +3,7 @@ import re import shutil from pathlib import Path -from typing import List +from typing import List, Union from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG @@ -26,34 +26,29 @@ templates = { ## Format example {format_example} ----- -Role: You are an architect; the goal is to perform incremental development and design a state-of-the-art (SOTA) PEP8-compliant Python system based on the context and the provided difference descriptions. Make the best use of good open source tools. -Requirement: Fill in the following missing information based on the context, each section name is a key in json +Role: You are an architect; the goal is to perform incremental development and design a state-of-the-art (SOTA) PEP8-compliant Python system based on the context and legacy design. Make the best use of good open source tools. +Requirement: Fill in the following missing information based on the context, each section name is a key in json. Output exactly as shown in the example, including single and double quotes. Max Output: 8192 chars or 2048 tokens. Try to use them up. -## Difference Description: Provided as Python list[str], the list of differences between the new design and the legacy design +## Difference Description: Provide as list, the foremost differences description for system design here based on the previous. -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Incremental implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. ## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## Data structures and interface definitions: Use single quotes to wrap content. Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. +## Program call flow: Use single quotes to wrap content. Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, -and only output the json inside this tag, nothing else +output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example. +Output exactly as shown in the example, including single and double quotes, and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] { "Difference Description": ["The ..."], - "Implementation approach": "We will ...", - "Python package name": "snake_game", - "File list": ["main.py"], + "Incremental implementation approach": "We will ...", + "Python package name": "new_name", "Data structures and interface definitions": ' classDiagram class Game{ @@ -67,8 +62,7 @@ and only output the json inside this tag, nothing else participant M as Main ... G->>M: end game - ', - "Anything UNCLEAR": "The requirement is clear to me." + ' } [/CONTENT] """, @@ -84,24 +78,20 @@ and only output the json inside this tag, nothing else ## Format example {format_example} ----- -Role: You are an architect; the goal is to perform incremental development and design a state-of-the-art (SOTA) PEP8-compliant Python system based on the context and the provided difference descriptions. Make the best use of good open source tools. -Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately +Role: You are an architect; the goal is to perform incremental development and design a state-of-the-art (SOTA) PEP8-compliant Python system based on the context and legacy design. Make the best use of good open source tools. +Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately. Output exactly as shown in the example, including single and double quotes. Max Output: 8192 chars or 2048 tokens. Try to use them up. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Difference Description: Provided as Python list[str], the list of differences between the new design and the legacy design +## Difference Description: Provide as list, the foremost differences description for system design here based on the previous. -## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Incremental implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. ## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores -## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here +## Data structures and interface definitions: Use single quotes to wrap content. Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. -## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design. - -## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. +## Program call flow: Use single quotes to wrap content. Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT. """, "FORMAT_EXAMPLE": """ @@ -113,19 +103,12 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ] ``` -## Implementation approach +## Incremental implementation approach We will ... ## Python package name ```python -"snake_game" -``` - -## File list -```python -[ - "main.py", -] +"new_name" ``` ## Data structures and interface definitions @@ -145,22 +128,19 @@ sequenceDiagram ... G->>M: end game ``` - -## Anything UNCLEAR -The requirement is clear to me. --- """, }, } OUTPUT_MAPPING = { - "Difference Description": (List[str], ...), - "Implementation approach": (str, ...), + # "Incremental Requirements": (str, ...), + "Difference Description": (Union[List[str], str], ...), + "Incremental implementation approach": (str, ...), "Python package name": (str, ...), - "File list": (List[str], ...), + # "File list": (List[str], ...), "Data structures and interface definitions": (str, ...), - "Program call flow": (str, ...), - "Anything UNCLEAR": (str, ...), + "Program call flow": (str, ...) } @@ -201,9 +181,6 @@ class RefineDesign(Action): async def _save_prd(self, docs_path, resources_path, context): prd_file = docs_path / "prd.md" - if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]: - quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"] - await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis") if context[-1].instruct_content: logger.info(f"Saving PRD to {prd_file}") diff --git a/metagpt/actions/refine_prd.py b/metagpt/actions/refine_prd.py index 81cb02af8..0f01b9904 100644 --- a/metagpt/actions/refine_prd.py +++ b/metagpt/actions/refine_prd.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Union from metagpt.actions import Refine, ActionOutput, SearchAndSummarize from metagpt.config import CONFIG @@ -9,96 +9,36 @@ increment_template = { "json": { "PROMPT_TEMPLATE": """ # Context -## User's New Requirements +## User's Incremental Requirements {new_requirements} -## Difference Description -{difference_description} - ## Legacy PRD {legacy} ## Search Information {search_information} -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` - ## Format example {format_example} ----- Role: You are a professional Product Manager tasked with overseeing incremental development and crafting Product Requirements Documents (PRDs) for a concise, usable, and efficient product. -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design +Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive designOnly output one json, nothing else. -## New Requirements: Provide as Plain text and place the new requirements here +## Incremental Requirements: Provide as str, the foremost incremental requirements for PRD here based on the previous. -## Difference Description: Provide as Python list[str], up to 5 clear, difference descriptions. If the requirement itself is simple, the difference description should also be simple +## Difference Description: Provide as str, the foremost differences description for PRD here based on the previous. ## Incremental Development Plan: Provide as Python list[str], up to 5 clear, incremental development plans. If the requirement itself is simple, the incremental development plan should also be simple -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. - output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": """ [CONTENT] { - "New Requirements": "", + "Incremental Requirements": "", "Difference Description": [], - "Search Information": "", - "Requirements": "", "Incremental Development Plan": [], - "Product Goals": [], - "User Stories": [], - "Competitive Analysis": [], - "Competitive Quadrant Chart": "quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - Campaign A: [0.3, 0.6] - Campaign B: [0.45, 0.23] - Campaign C: [0.57, 0.69] - Campaign D: [0.78, 0.34] - Campaign E: [0.40, 0.34] - Campaign F: [0.35, 0.78]", - "Requirement Analysis": "", - "Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], - "UI Design draft": "", - "Anything UNCLEAR": "", } [/CONTENT] """, @@ -106,154 +46,58 @@ and only output the json inside this tag, nothing else "markdown": { "PROMPT_TEMPLATE": """ # Context -You need to refine the requirements based on the new requirements and the existing requirements' output. -## User's New Requirements +You need to refine the requirements based on the Incremental Requirements and the existing requirements' output. +## User's Incremental Requirements {new_requirements} -## Difference Description -{difference_description} - ## Legacy PRD {legacy} ## Search Information {search_information} -## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the with REAL COMPETITOR NAME -```mermaid -quadrantChart - title Reach and engagement of campaigns - x-axis Low Reach --> High Reach - y-axis Low Engagement --> High Engagement - quadrant-1 We should expand - quadrant-2 Need to promote - quadrant-3 Re-evaluate - quadrant-4 May be improved - "Campaign: A": [0.3, 0.6] - "Campaign B": [0.45, 0.23] - "Campaign C": [0.57, 0.69] - "Campaign D": [0.78, 0.34] - "Campaign E": [0.40, 0.34] - "Campaign F": [0.35, 0.78] - "Our Target Product": [0.5, 0.6] -``` - ## Format example {format_example} ----- Role: You are a professional Product Manager tasked with overseeing incremental development and crafting Product Requirements Documents (PRDs) for a concise, usable, and efficient product. Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format.Only output one json, nothing else. -## New Requirements: Provide as Plain text and place the new requirements here +## Incremental Requirements: Provide as str, the foremost incremental requirements for PRD here based on the previous. -## Difference Description: Provide as Python list[str], up to 5 clear, difference descriptions. If the requirement itself is simple, the difference description should also be simple +## Difference Description: Provide as str, the foremost differences description for PRD here based on the previous. ## Incremental Development Plan: Provide as Python list[str], up to 5 clear, incremental development plans. If the requirement itself is simple, the incremental development plan should also be simple - -## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple - -## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less - -## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible - -## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible. - -## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery. - -## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower - -## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description. -## Anything UNCLEAR: Provide as Plain text. Make clear here. """, "FORMAT_EXAMPLE": """ --- -## New Requirements +## Incremental Requirements The boss ... ## Difference Description -```python -[ - "...", -] +... ## Incremental Development Plan [ - "It ...", + "...", ] - -## Product Goals -```python -[ - "Create a ...", -] -``` - -## User Stories -```python -[ - "As a user, ...", -] -``` - -## Competitive Analysis -```python -[ - "Python Snake Game: ...", -] -``` - -## Competitive Quadrant Chart -```mermaid -quadrantChart - title Reach and engagement of campaigns - ... - "Our Target Product": [0.6, 0.7] -``` - -## Requirement Analysis -The product should be a ... - -## Requirement Pool -```python -[ - ["End game ...", "P0"] -] -``` - -## UI Design draft -Give a basic function description, and a draft - -## Anything UNCLEAR -There are no unclear points. ---- """, }, } INCREMENT_OUTPUT_MAPPING = { - "New Requirements": (str, ...), - # "Major Enhancements": (List[str], ...), - "Difference Description": (List[str], ...), + "Incremental Requirements": (str, ...), + "Difference Description": (Union[List[str], str], ...), "Incremental Development Plan": (List[str], ...), - "Product Goals": (List[str], ...), - "User Stories": (List[str], ...), - "Competitive Analysis": (List[str], ...), - "Competitive Quadrant Chart": (str, ...), - "Requirement Analysis": (str, ...), - "Requirement Pool": (List[List[str]], ...), - "UI Design draft": (str, ...), - "Anything UNCLEAR": (str, ...), } -# 对于产品经理，增量开发的动作是：RefinePDR，输出是结合新需求和已有需求输出的新的PDR class RefinePRD(Refine): def __init__(self, name="RefinePRD", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, new_requirements, difference_description, legacy, format=CONFIG.prompt_format, *args, **kwargs): + async def run(self, new_requirements, legacy, format=CONFIG.prompt_format, *args, **kwargs): sas = SearchAndSummarize() rsp = "" info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" @@ -263,7 +107,7 @@ class RefinePRD(Refine): prompt_template, format_example = get_template(increment_template, format) prompt = prompt_template.format( - new_requirements=new_requirements, difference_description=difference_description, legacy=legacy, search_information=info, + new_requirements=new_requirements, legacy=legacy, search_information=info, format_example=format_example ) logger.debug(prompt) diff --git a/metagpt/actions/refine_project_management.py b/metagpt/actions/refine_project_management.py index 7d9e118d8..dfeeb2db0 100644 --- a/metagpt/actions/refine_project_management.py +++ b/metagpt/actions/refine_project_management.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from typing import List +from typing import List, Union from metagpt.actions.action import Action from metagpt.config import CONFIG @@ -15,62 +15,50 @@ templates = { # Context {context} -## Legacy Design +## Legacy {legacy} ## Format example {format_example} ----- -Role: You are a project manager; the goal is to perform incremental development based on the context and difference descriptions and the legacy. Break down tasks according to PRD/technical design, provide a task list, and analyze task dependencies to start with the prerequisite modules. -Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Role: You are a project manager; the goal is to perform incremental development based on the context and difference descriptions and the legacy. Break down tasks according to PRD/technical design, provide a Task list, and analyze task dependencies to start with the prerequisite modules. +Requirements: Based on the context and the Legacy Project Management and Legacy Code, fill in the following missing information. Note that Please try your best to reuse legacy code, and all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file that need to modified. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Required Python third-party packages: Provided in requirements.txt format +## Difference Description: Provide as a python list, the foremost differences description for project management here based on the previous. -## Required Other language third-party packages: Provided in requirements.txt format +## Incremental Required Python third-party packages: Provided as a python list, the requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend based on the previous. -## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. +## Logic Analysis: Only files need to modified, Provided as a Python list[list[str]. If the file has no changes, the file will not be output. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first based on the previous. -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Task list: Only files need to modified, provided as Python list[str]. If the file has no changes, the file will not be output. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": ''' { - "Required Python third-party packages": [ + "Incremental Requirements": "...", + "Difference Description": [ + "...", + ] + "Incremental Required Python third-party packages": [ "flask==1.1.2", "bcrypt==3.2.0" ], - "Required Other language third-party packages": [ - "No third-party ..." - ], "Full API spec": """ openapi: 3.0.0 ... description: A JSON object ... """, - "Difference Description": """ - The ... - """, "Logic Analysis": [ ["game.py","Contains..."] ], "Task list": [ "game.py" - ], - "Shared Knowledge": """ - 'game.py' contains ... - """, - "Anything UNCLEAR": "We need ... how to start." + ] } ''', }, @@ -79,48 +67,44 @@ and only output the json inside this tag, nothing else # Context {context} -## Legacy Design +## Legacy {legacy} ## Format example {format_example} ----- -Role: You are a project manager; the goal is to perform incremental development based on the context and difference descriptions and the legacy. Break down tasks according to PRD/technical design, provide a task list, and analyze task dependencies to start with the prerequisite modules. -Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them +Role: You are a project manager; the goal is to perform incremental development based on the context and the legacy. Break down tasks according to PRD/technical design, provide a Task list need to modified files, and analyze task dependencies to start with the prerequisite modules. +Requirements: Based on the context and the Legacy Project Management and Legacy Code, fill in the following missing information. Note that Please try your best to reuse legacy code, and all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file that need to modified. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Required Python third-party packages: Provided in requirements.txt format +## Difference Description: Provided as a python list, the foremost differences description for project management here based on the previous. -## Required Other language third-party packages: Provided in requirements.txt format +## Incremental Required Python third-party packages: Provided as a python list, the requirements.txt format -## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend. +## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend based on the previous. -## Difference Description: Please provide a detailed description of the differences between this project and its predecessors or similar projects that can include changes in technology, architecture. - -## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first - -## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first. - -## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. +## Logic Analysis: Only files need to modified, Provided as a Python list[list[str]. If the file has no changes, the file will not be output. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first based on the previous. +## Task list: Only files need to modified, provided as Python list[str]. If the file has no changes, the file will not be output. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first """, "FORMAT_EXAMPLE": ''' --- -## Required Python third-party packages +## Incremental Requirements +... + +## Difference Description ```python -""" -flask==1.1.2 -bcrypt==3.2.0 -""" +[ + "The ...", +] ``` -## Required Other language third-party packages +## Incremental Required Python third-party packages ```python -""" -No third-party ... -""" +[ + "flask==1.1.2", + "bcrypt==3.2.0" +] ``` ## Full API spec @@ -132,13 +116,6 @@ description: A JSON object ... """ ``` -## Difference Description -```python -""" -The ... -""" -``` - ## Logic Analysis ```python [ @@ -152,29 +129,18 @@ The ... "game.py", ] ``` - -## Shared Knowledge -```python -""" -'game.py' contains ... -""" -``` - -## Anything UNCLEAR -We need ... how to start. --- ''', }, } OUTPUT_MAPPING = { - "Required Python third-party packages": (List[str], ...), - "Required Other language third-party packages": (List[str], ...), + # "Incremental Requirements": (str, ...), + # ## Incremental Requirements: Provided as a str, the foremost incremental requirements for project management here based on the previous. + "Difference Description": (Union[List[str], str], ...), + "Incremental Required Python third-party packages": (Union[List[str], str], ...), "Full API spec": (str, ...), - "Difference Description": (str, ...), "Logic Analysis": (List[List[str]], ...), "Task list": (List[str], ...), - "Shared Knowledge": (str, ...), - "Anything UNCLEAR": (str, ...), } @@ -192,11 +158,13 @@ class RefineTasks(Action): # Write requirements.txt requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" - requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) + requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Incremental Required Python third-party packages"))) async def run(self, context, legacy, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) - prompt = prompt_template.format(context=context, legacy=legacy, format_example=format_example) + prompt = prompt_template.format(context=context, + legacy=legacy, + format_example=format_example) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format) self._save(context, rsp) return rsp diff --git a/metagpt/actions/write_code_guide.py b/metagpt/actions/write_code_guide.py new file mode 100644 index 000000000..391173bb3 --- /dev/null +++ b/metagpt/actions/write_code_guide.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from metagpt.actions.action import Action +from metagpt.logs import logger +from metagpt.schema import Message +from metagpt.utils.common import CodeParser +from tenacity import retry, stop_after_attempt, wait_fixed + + +PROMPT_TEMPLATE = """ +NOTICE +Role: You are a professional software engineer, and your main task is to conduct incremental development, proposing incremental development plans and code guideance based on context and legacy code. Existing code and logic that need to be retained must also appear in the code after incremental development, do not omit it. Ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". + +## Regulations Review: To make the software directly operable without further coding, follow the regulations below during incremental development: +1) Import all referenced classes. +2) Implement all methods. +3) Add necessary explanation to all methods. +4) Ensure there are no potential bugs. +5) Confirm that the entire project conforms to the tasks proposed by the user. +6) Review the code thoroughly, checking for errors and validating the logic to ensure seamless user interaction without compromising any specified requirements. + +## Incremental Development Plan: Proposed the Minimum essential incremental development plan, based on the following context and legacy code by thinking step by step. +... + +## Code guidelines: Propose the foremost guidelines that how to implement code of modification part for incremental development based on the above context, legacy code and incremental development plan. +```python +... +''' +----- +# Context +{context} + +## Legacy Code +You are tasked with conducting incremental development in the existing code based on the provided legacy code and above information. +``` +{code} +``` +----- + +## Format example +----- +## Incremental Development Guide: +... + +## Code Guidance: +# Implementation the ... +```python +... +''' +----- +""" + + +class WriteCodeGuide(Action): + def __init__(self, name="WriteCodeGuide", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) + + async def run(self, context, code): + prompt = PROMPT_TEMPLATE.format(context=context, code=code) + logger.info(f'Write Code Guide ..') + code_guide = await self._aask(prompt) + # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) + # self._save(context, filename, code) + return code_guide + \ No newline at end of file diff --git a/metagpt/actions/write_code_refine.py b/metagpt/actions/write_code_refine.py index 527952aed..466c30679 100644 --- a/metagpt/actions/write_code_refine.py +++ b/metagpt/actions/write_code_refine.py @@ -7,58 +7,44 @@ from metagpt.schema import Message from metagpt.utils.common import CodeParser from tenacity import retry, stop_after_attempt, wait_fixed - PROMPT_TEMPLATE = """ NOTICE -Role: You are a professional software engineer, and your main task is to conduct incremental development, which includes reviewing existing code, providing modification suggestions, rewriting code, and optimizing the codebase. Existing code and logic that need to be retained must also appear in the code after incremental development, do not omit it. Ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". +Role: You are a professional engineer; your primary goal is to write PEP8 compliant, elegant, modular, easy-to-read, and maintainable Python 3.9 code (or any other programming language of your choice). +Requirements: You should modify the corresponding code based on the guidance. Then, output the complete code, fixing all errors according to the context. Ensure that you adhere to the specified guidelines for incremental development and modification of legacy code. +ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format should be carefully referenced using the "Format example". Only output the current modified code, nothing else. In the modified code, if unchanged, you should output it, the complete code. -## Code Review: Based on the following context and legacy code, and following the checklist, provide key, clear, concise, and specific code modification suggestions, up to 5. -``` -1. Check 0: Is the code implemented as per the requirements? -2. Check 1: Are there any issues with the code logic? -3. Check 2: Does the existing code follow the "Data structures and interface definitions"? -4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented? -5. Check 4: Does the code have unnecessary or lack dependencies? -``` - -## Incremental Development: {filename} Based on the findings from the "Code Review," context, and the legacy code, conduct incremental development by rewriting, optimizing, and adding new code using triple quotes. +## Code: Only Write {filename}, Write code using triple quotes, based on the following list and context. +1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Requirement: Implement one of the following code files based on the provided context. Return the code in the specified format. Your code will be part of the entire project, so ensure it is complete, reliable, and reusable. +3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. +5. Think before writing: What should be implemented and provided in this document? +6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. +7. Do not use public member functions that do not exist in your design. ----- # Context {context} -## Legacy Code -You are tasked with conducting incremental development in the existing code and creating a new code file, {filename}, based on the provided legacy code and above information. -``` -{code} -``` ----- +## Guidelines: The foremost guidelines of modification for incremental development. +{guide} +----- +## Legacy Code: The Legacy Code that needs to be modified. +{legacy} + +----- ## Format example ----- -{format_example} ------ - -""" - -FORMAT_EXAMPLE = """ - -## Code Review -1. The code ... -2. ... -3. ... -4. ... -5. ... - -## Incremental Development: {filename} +## Modified/Added Code: {filename} ```python -## {filename} - Incremental Development +# {filename} ... ``` +----- """ - class WriteCodeRefine(Action): def __init__(self, name="WriteCodeRefine", context: list[Message] = None, llm=None): super().__init__(name, context, llm) @@ -69,9 +55,8 @@ class WriteCodeRefine(Action): code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, code, filename): - format_example = FORMAT_EXAMPLE.format(filename=filename) - prompt = PROMPT_TEMPLATE.format(context=context, code=code, filename=filename, format_example=format_example) + async def run(self, context, code, filename, guide): + prompt = PROMPT_TEMPLATE.format(context=context, legacy=code, filename=filename, guide=guide) logger.info(f'Code refine {filename}..') code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) diff --git a/metagpt/environment.py b/metagpt/environment.py index 24e6ada2f..4e409b921 100644 --- a/metagpt/environment.py +++ b/metagpt/environment.py @@ -24,6 +24,7 @@ class Environment(BaseModel): roles: dict[str, Role] = Field(default_factory=dict) memory: Memory = Field(default_factory=Memory) history: str = Field(default='') + legacy: dict = Field(default_factory=dict) class Config: arbitrary_types_allowed = True @@ -77,3 +78,15 @@ class Environment(BaseModel): get all the environment roles """ return self.roles.get(name, None) + + def set_legacy(self, legacy_dict: dict) -> None: + """设置环境的遗产 + set the environment legacy + """ + self.legacy = legacy_dict + + def get_legacy(self) -> dict: + """获得环境的遗产 + get the environment legacy + """ + return self.legacy diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index dddcd2a8b..f87f8899d 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -32,12 +32,10 @@ class Architect(Role): profile: str = "Architect", goal: str = "Design a concise, usable, complete python system", constraints: str = "Try to specify good open source tools as much as possible", - legacy: str = "", increment: bool = False, ) -> None: """Initializes the Architect with given attributes.""" super().__init__(name, profile, goal, constraints) - self.legacy = legacy self.increment = increment # Initialize actions specific to the Architect role @@ -52,7 +50,8 @@ class Architect(Role): async def _act(self) -> Message: if self.increment: logger.info(f"{self._setting}: ready to RefineDesign") - response = await self._rc.todo.run(self._rc.history, self.legacy) + legacy = self._rc.env.get_legacy()["legacy_design"] + response = await self._rc.todo.run(self._rc.history, legacy) else: logger.info(f"{self._setting}: ready to WriteDesign") diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 13034acaa..6f96aeb78 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -14,8 +14,10 @@ from typing import List from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks, BossRequirement from metagpt.actions.refine_design_api import RefineDesign +from metagpt.actions.refine_prd import RefinePRD from metagpt.actions.refine_project_management import RefineTasks from metagpt.actions.write_code_refine import WriteCodeRefine +from metagpt.actions.write_code_guide import WriteCodeGuide from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.roles import Role @@ -73,31 +75,25 @@ class Engineer(Role): constraints: str = "The code should conform to standards like PEP8 and be modular and maintainable", n_borg: int = 1, use_code_review: bool = False, - legacy: str = "", increment: bool = False, - bug_msgs: List = None, bug_fix: bool = False, ) -> None: """Initializes the Engineer role with given attributes.""" super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review - self.legacy = legacy self.increment = increment - self.bug_msgs = bug_msgs self.bug_fix = bug_fix - if self.use_code_review or self.increment: - self._init_actions([WriteCode, WriteCodeReview, WriteCodeRefine]) + if self.use_code_review: + self._init_actions([WriteCode, WriteCodeReview]) if self.increment: + self._init_actions([WriteCodeGuide, WriteCodeRefine, WriteCodeReview]) self._watch([RefineTasks]) - self.todos = [] - elif self.bug_fix: - self._watch([BossRequirement]) - self.todos = [] else: self._watch([WriteTasks]) - self.todos = [] + + self.todos = [] self.n_borg = n_borg @classmethod @@ -210,36 +206,50 @@ class Engineer(Role): ) return msg - async def _act_increment(self, legacy) -> Message: + async def _act_increment(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later workspace = self.get_workspace() - flag = True - # legacy_codes = legacy.split('---') + # human_str = "\n".join([msg.content for msg in self._rc.memory.get_by_role("Human")]) + human_str = str(self._rc.memory.get_by_role("Human")[0]) + code = self._rc.env.get_legacy()["legacy_code"] + + # Refine code + context = [] + msg = self._rc.memory.get_by_actions([RefinePRD, RefineDesign, RefineTasks]) + + for m in msg: + context.append(m.content) + context_str = human_str + "\n".join(context) + try: + logger.info("Write Code Guide start!") + guide = await WriteCodeGuide().run(context=context_str, code=code) + msg = Message(content=guide, role=self.profile, cause_by=WriteCodeGuide) + self._rc.memory.add(msg) + except Exception as e: + logger.error("Write Code Guide failed!", e) + pass + + # Write code or Code review for todo in self.todos: - context = [] - - msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks, WriteCodeRefine]) - for m in msg: - context.append(m.content) - context_str = "\n".join(context) - code = legacy - - # Refine code or Write code - # if self.increment and len(legacy_codes) > 0: - # code = legacy_codes.pop(0) - - # Code review + msg = self._rc.memory.get_by_actions([RefineTasks]) + context_str = human_str + "\n".join([m.content for m in msg]) + # WriteCodeRefine try: - rewrite_code = await WriteCodeRefine().run(context=context_str, code=code, filename=todo) - code = rewrite_code + logger.info("Write Code Refine start!") + code = await WriteCodeRefine().run(context=context_str, code=code, filename=todo, guide=guide) except Exception as e: - logger.error("code review failed!", e) + logger.error("Write Code Refine failed!", e) pass - - # code = await WriteCode().run(context=context_str, filename=todo) - + # FIXME: Code review Action + # if self.use_code_review: + # try: + # rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) + # code = rewrite_code + # except Exception as e: + # logger.error("code review failed!", e) + # pass file_path = self.write_file(workspace, todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) + msg = Message(content=code, role=self.profile, cause_by=WriteCodeRefine) self._rc.memory.add(msg) code_msg = todo + FILENAME_CODE_SEP + str(file_path) @@ -273,7 +283,7 @@ class Engineer(Role): # Code review try: - rewrite_code = await WriteCodeRefine().run(context=context_str, code=code, filename=todo) + rewrite_code = await WriteCodeGuide().run(context=context_str, code=code, filename=todo) code = rewrite_code except Exception as e: logger.error("code review failed!", e) @@ -333,35 +343,31 @@ class Engineer(Role): ) return msg - async def _observe(self) -> int: - if self.bug_fix: - msg = Message( - content=self.bug_msgs[0].content + "\n---\n" + self.legacy, - role=self.profile, - cause_by=BossRequirement, - sent_from=self.profile, - send_to=self.profile, - ) - self._publish_message(msg) - await super()._observe() - self._rc.news = [ - msg for msg in self._rc.news if msg.send_to == self.profile - ] # only relevant msgs count as observed news - return len(self._rc.news) + # async def _observe(self) -> int: + # if self.bug_fix: + # msg = Message( + # content=self.bug_msgs[0].content + "\n---\n" + self.legacy, + # role=self.profile, + # cause_by=BossRequirement, + # sent_from=self.profile, + # send_to=self.profile, + # ) + # self._publish_message(msg) + # await super()._observe() + # self._rc.news = [ + # msg for msg in self._rc.news if msg.send_to == self.profile + # ] # only relevant msgs count as observed news + # return len(self._rc.news) async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" if self.increment: - logger.info(f"{self._setting}: ready to RefineWriteCode") - elif self.bug_fix: - logger.info(f"{self._setting}: ready to BugFix") + logger.info(f"{self._setting}: ready to WriteExtraCode and WriteCodeRefine") else: logger.info(f"{self._setting}: ready to WriteCode") - if self.use_code_review: + if self.increment: + return await self._act_increment() + elif self.use_code_review: return await self._act_sp_precision() - elif self.increment: - return await self._act_increment(self.legacy) - elif self.bug_fix: - return await self._act_bug_fix(self.bug_msgs) return await self._act_sp() diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index 0b2d83ed0..f48720430 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -29,8 +29,6 @@ class ProductManager(Role): profile: str = "Product Manager", goal: str = "Efficiently create a successful product", constraints: str = "", - difference_description: str = "", - legacy: str = "", increment: bool = False, ) -> None: """ @@ -43,8 +41,6 @@ class ProductManager(Role): constraints (str): Constraints or limitations for the product manager. """ super().__init__(name, profile, goal, constraints) - self.difference_description = difference_description - self.legacy = legacy self.increment = increment if self.increment: @@ -56,7 +52,8 @@ class ProductManager(Role): async def _act(self) -> Message: if self.increment: logger.info(f"{self._setting}: ready to RefinePRD") - response = await self._rc.todo.run(self._rc.history, self.difference_description, self.legacy) + legacy = self._rc.env.get_legacy()["legacy_prd"] + response = await self._rc.todo.run(self._rc.history, legacy) else: logger.info(f"{self._setting}: ready to WritePRD") diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 21a196d21..5d4820f4a 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -32,7 +32,6 @@ class ProjectManager(Role): goal: str = "Improve team efficiency and deliver with quality and quantity", constraints: str = "", increment: bool = False, - legacy: str = "", ) -> None: """ Initializes the ProjectManager role with given attributes. @@ -45,8 +44,6 @@ class ProjectManager(Role): """ super().__init__(name, profile, goal, constraints) self.increment = increment - self.legacy = legacy - if self.increment: self._init_actions([RefineTasks]) self._watch([RefineDesign]) @@ -57,7 +54,11 @@ class ProjectManager(Role): async def _act(self) -> Message: if self.increment: logger.info(f"{self._setting}: ready to RefineTasks") - response = await self._rc.todo.run(self._rc.history, self.legacy) + human_str = "\n".join([msg.content for msg in self._rc.memory.get_by_role("Human")]) + # legacy_project_management and legacy_code + legacy_dict = self._rc.env.get_legacy() + legacy_str = "Legacy Project Management:\n" + legacy_dict["legacy_project_management"] + "\nLegacy Code:\n" + legacy_dict["legacy_code"] + response = await self._rc.todo.run(self._rc.history, legacy=legacy_str) else: logger.info(f"{self._setting}: ready to WriteTasks") diff --git a/metagpt/team.py b/metagpt/team.py index 67d3ecec8..67f1973b5 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -59,4 +59,10 @@ class Team(BaseModel): self._check_balance() await self.environment.run() return self.environment.history + + def set_legacy(self, legacy_dict): + self.environment.legacy = legacy_dict + + def get_legacy(self): + return self.environment.legacy \ No newline at end of file diff --git a/startup.py b/startup.py index 8bb8517c9..647fc307e 100644 --- a/startup.py +++ b/startup.py @@ -28,12 +28,11 @@ async def startup( run_tests: bool = False, implement: bool = True, increment: bool = False, - bug_fix: bool = False, ): """Run a startup. Be a boss.""" company = Team() - if increment or bug_fix: + if increment: # 读取文件 prd_path = os.path.join(project_path, 'docs/prd.md') design_path = os.path.join(project_path, 'docs/system_design.md') @@ -47,64 +46,46 @@ async def startup( legacy_design = f.read() with open(api_spec_and_tasks_path, 'r', encoding='utf-8') as f: - legacy_api_spec_and_tasks = f.read() + legacy_project_management = f.read() - # 遍历文件夹，获取所有代码文件 legacy_code = '' for root, dirs, files in os.walk(code_path): - filenames = [filename for filename in files if filename.endswith('.py')] - legacy_code += f'There are {len(files)} scripts in the current folder: {", ".join(filenames)}\n---\n' for file in files: if file.endswith('.py'): with open(os.path.join(root, file), 'r', encoding='utf-8') as f: legacy_code += f.read() + '\n---\n' - if bug_fix: - boss_msg = Message( - content=f"Boss's requirement\n:{idea}\n---\nBoss's difference description:{difference_description}\n---\n", - sent_from="Boss", - ) - product_manager_msg = Message( - content=f"Product Manager's prd legacy:\n{legacy_prd}\n---\n", - sent_from="ProductManager" - ) - architect_msg = Message( - content=f"Architect's design legacy:\n{legacy_design}\n---\n", - sent_from="Architect" - ) - project_manager_msg = Message( - content=f"Project Manager's api spec and tasks legacy:\n{legacy_api_spec_and_tasks}\n---\n", - sent_from="ProjectManager" - ) - engineer_msg = Message( - content=f"Engineer's code legacy:\n{legacy_code}\n---\n", - sent_from="Engineer" - ) - bug_msgs = [boss_msg, product_manager_msg, architect_msg, project_manager_msg, engineer_msg] - else: - company.hire( - [ProductManager(difference_description=difference_description, legacy=legacy_prd, increment=increment), - Architect(legacy=legacy_design, increment=increment), - ProjectManager(legacy=legacy_api_spec_and_tasks, increment=increment)]) + legacy_dict = { + 'legacy_prd': legacy_prd, + 'legacy_design': legacy_design, + 'legacy_project_management': legacy_project_management, + 'legacy_code': legacy_code + } + company.set_legacy(legacy_dict) + company.hire( + [ + ProductManager(increment=increment), + Architect(increment=increment), + ProjectManager(increment=increment) + ] + ) else: company.hire([ProductManager(), Architect(), ProjectManager()]) # if implement or code_review - if bug_fix: - company.hire([Engineer(n_borg=5, bug_msgs=bug_msgs, bug_fix=bug_fix)]) - elif implement or code_review: + if (implement or code_review) and increment: + company.hire([Engineer(n_borg=5, use_code_review=code_review, increment=increment)]) + elif implement: # developing features: implement the idea company.hire([Engineer(n_borg=5, use_code_review=code_review)]) - elif (implement or code_review) and increment: - company.hire([Engineer(n_borg=5, use_code_review=code_review, legacy=legacy_code, increment=increment)]) - if run_tests or bug_fix: + if run_tests: # developing features: run tests on the spot and identify bugs # (bug fixing capability comes soon!) company.hire([QaEngineer()]) company.invest(investment) - company.start_project(idea) + company.start_project(idea+"\n"+difference_description) await company.run(n_round=n_round) @@ -118,7 +99,6 @@ def main( run_tests: bool = False, implement: bool = True, increment: bool = False, - bug_fix: bool = False, ): """ We are a software startup comprised of AI. By investing in us, @@ -130,10 +110,8 @@ def main( :param code_review: Whether to use code review. :return: """ - - asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy()) asyncio.run( - startup(idea, difference_description, project_path, investment, n_round, code_review, run_tests, implement, increment, bug_fix)) + startup(idea, difference_description, project_path, investment, n_round, code_review, run_tests, implement, increment)) if __name__ == "__main__": From 452cdb7ff6d1fa6a855820530c51e71f481eea30 Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Wed, 29 Nov 2023 17:56:30 +0800 Subject: [PATCH 5/6] update increment development, add write_code_guide.py file --- metagpt/roles/engineer.py | 19 +------------------ metagpt/team.py | 6 ------ startup.py | 2 +- 3 files changed, 2 insertions(+), 25 deletions(-) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 6f96aeb78..66ce12f83 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -209,13 +209,12 @@ class Engineer(Role): async def _act_increment(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later workspace = self.get_workspace() - # human_str = "\n".join([msg.content for msg in self._rc.memory.get_by_role("Human")]) human_str = str(self._rc.memory.get_by_role("Human")[0]) code = self._rc.env.get_legacy()["legacy_code"] # Refine code context = [] - msg = self._rc.memory.get_by_actions([RefinePRD, RefineDesign, RefineTasks]) + msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks]) for m in msg: context.append(m.content) @@ -343,22 +342,6 @@ class Engineer(Role): ) return msg - # async def _observe(self) -> int: - # if self.bug_fix: - # msg = Message( - # content=self.bug_msgs[0].content + "\n---\n" + self.legacy, - # role=self.profile, - # cause_by=BossRequirement, - # sent_from=self.profile, - # send_to=self.profile, - # ) - # self._publish_message(msg) - # await super()._observe() - # self._rc.news = [ - # msg for msg in self._rc.news if msg.send_to == self.profile - # ] # only relevant msgs count as observed news - # return len(self._rc.news) - async def _act(self) -> Message: """Determines the mode of action based on whether code review is used.""" if self.increment: diff --git a/metagpt/team.py b/metagpt/team.py index 67f1973b5..ea5de4a12 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -60,9 +60,3 @@ class Team(BaseModel): await self.environment.run() return self.environment.history - def set_legacy(self, legacy_dict): - self.environment.legacy = legacy_dict - - def get_legacy(self): - return self.environment.legacy - \ No newline at end of file diff --git a/startup.py b/startup.py index 647fc307e..60298ed36 100644 --- a/startup.py +++ b/startup.py @@ -61,7 +61,7 @@ async def startup( 'legacy_project_management': legacy_project_management, 'legacy_code': legacy_code } - company.set_legacy(legacy_dict) + company.environment.set_legacy(legacy_dict) company.hire( [ ProductManager(increment=increment), From acb70fa30c19d068f39f3a295140436bcad7ee82 Mon Sep 17 00:00:00 2001 From: mannaandpoem <1580466765@qq.com> Date: Mon, 4 Dec 2023 17:14:52 +0800 Subject: [PATCH 6/6] update increment development --- metagpt/actions/refine_design_api.py | 31 +++----- metagpt/actions/refine_prd.py | 46 ++++------- metagpt/actions/refine_project_management.py | 58 ++++---------- metagpt/actions/write_code_guide.py | 45 ++++++----- metagpt/actions/write_code_refine.py | 22 +++--- metagpt/roles/architect.py | 6 +- metagpt/roles/engineer.py | 82 +++++++------------- metagpt/roles/product_manager.py | 4 +- metagpt/roles/project_manager.py | 5 +- metagpt/team.py | 30 +++++++ startup.py | 42 ++-------- 11 files changed, 153 insertions(+), 218 deletions(-) diff --git a/metagpt/actions/refine_design_api.py b/metagpt/actions/refine_design_api.py index 909fa6db9..d6a948b43 100644 --- a/metagpt/actions/refine_design_api.py +++ b/metagpt/actions/refine_design_api.py @@ -20,7 +20,7 @@ templates = { # Context {context} -## Legacy Design +## Legacy {legacy} ## Format example @@ -30,9 +30,7 @@ Role: You are an architect; the goal is to perform incremental development and d Requirement: Fill in the following missing information based on the context, each section name is a key in json. Output exactly as shown in the example, including single and double quotes. Max Output: 8192 chars or 2048 tokens. Try to use them up. -## Difference Description: Provide as list, the foremost differences description for system design here based on the previous. - -## Incremental implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Incremental implementation approach: Provide as Python list[str]. Analyze the difficult points of the requirements, select the appropriate open-source framework. Up to 5. ## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores @@ -46,8 +44,7 @@ Output exactly as shown in the example, including single and double quotes, and "FORMAT_EXAMPLE": """ [CONTENT] { - "Difference Description": ["The ..."], - "Incremental implementation approach": "We will ...", + "Incremental implementation approach": ["We will ...",], "Python package name": "new_name", "Data structures and interface definitions": ' classDiagram @@ -72,7 +69,7 @@ Output exactly as shown in the example, including single and double quotes, and # Context {context} -## Legacy Design +## Legacy {legacy} ## Format example @@ -83,9 +80,7 @@ Requirement: Fill in the following missing information based on the context, not Max Output: 8192 chars or 2048 tokens. Try to use them up. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Difference Description: Provide as list, the foremost differences description for system design here based on the previous. - -## Incremental implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework. +## Incremental implementation approach: Provide as Python list[str]. Analyze the difficult points of the requirements, select the appropriate open-source framework. Up to 5. ## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores @@ -96,15 +91,13 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W """, "FORMAT_EXAMPLE": """ --- -## Difference Description -```python -[ - "The ...", -] -``` ## Incremental implementation approach -We will ... +```python +[ + "We will ...", +] +``` ## Python package name ```python @@ -135,8 +128,8 @@ sequenceDiagram OUTPUT_MAPPING = { # "Incremental Requirements": (str, ...), - "Difference Description": (Union[List[str], str], ...), - "Incremental implementation approach": (str, ...), + # "Difference Description": (Union[List[str], str], ...), + "Incremental implementation approach": (Union[List[str], str], ...), "Python package name": (str, ...), # "File list": (List[str], ...), "Data structures and interface definitions": (str, ...), diff --git a/metagpt/actions/refine_prd.py b/metagpt/actions/refine_prd.py index 0f01b9904..1d8bab5f8 100644 --- a/metagpt/actions/refine_prd.py +++ b/metagpt/actions/refine_prd.py @@ -9,10 +9,9 @@ increment_template = { "json": { "PROMPT_TEMPLATE": """ # Context -## User's Incremental Requirements -{new_requirements} +{context} -## Legacy PRD +## Legacy {legacy} ## Search Information @@ -22,13 +21,9 @@ increment_template = { {format_example} ----- Role: You are a professional Product Manager tasked with overseeing incremental development and crafting Product Requirements Documents (PRDs) for a concise, usable, and efficient product. -Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive designOnly output one json, nothing else. +Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design. Only output one json, nothing else. -## Incremental Requirements: Provide as str, the foremost incremental requirements for PRD here based on the previous. - -## Difference Description: Provide as str, the foremost differences description for PRD here based on the previous. - -## Incremental Development Plan: Provide as Python list[str], up to 5 clear, incremental development plans. If the requirement itself is simple, the incremental development plan should also be simple +## Incremental Development Analysis: Provide as Python list[str], up to 5. incremental development analysis and plans based on the context and the legacy. If the requirement itself is simple, the Incremental Development Analysis should also be simple. output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else @@ -36,9 +31,7 @@ and only output the json inside this tag, nothing else "FORMAT_EXAMPLE": """ [CONTENT] { - "Incremental Requirements": "", - "Difference Description": [], - "Incremental Development Plan": [], + "Incremental Development Analysis": [], } [/CONTENT] """, @@ -46,11 +39,9 @@ and only output the json inside this tag, nothing else "markdown": { "PROMPT_TEMPLATE": """ # Context -You need to refine the requirements based on the Incremental Requirements and the existing requirements' output. -## User's Incremental Requirements -{new_requirements} +{context} -## Legacy PRD +## Legacy {legacy} ## Search Information @@ -63,32 +54,21 @@ Role: You are a professional Product Manager tasked with overseeing incremental Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## ' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format.Only output one json, nothing else. -## Incremental Requirements: Provide as str, the foremost incremental requirements for PRD here based on the previous. - -## Difference Description: Provide as str, the foremost differences description for PRD here based on the previous. - -## Incremental Development Plan: Provide as Python list[str], up to 5 clear, incremental development plans. If the requirement itself is simple, the incremental development plan should also be simple +## Incremental Development Analysis: Provide as Python list[str], up to 5. Incremental development analysis and plans based on the context and the legacy. If the requirement itself is simple, the Incremental Development Analysis should also be simple. """, "FORMAT_EXAMPLE": """ --- -## Incremental Requirements -The boss ... -## Difference Description -... - -## Incremental Development Plan +## Incremental Development Analysis [ - "...", + "We will ...", ] """, }, } INCREMENT_OUTPUT_MAPPING = { - "Incremental Requirements": (str, ...), - "Difference Description": (Union[List[str], str], ...), - "Incremental Development Plan": (List[str], ...), + "Incremental Development Analysis": (List[str], ...), } @@ -97,7 +77,7 @@ class RefinePRD(Refine): def __init__(self, name="RefinePRD", context=None, llm=None): super().__init__(name, context, llm) - async def run(self, new_requirements, legacy, format=CONFIG.prompt_format, *args, **kwargs): + async def run(self, context, legacy, format=CONFIG.prompt_format, *args, **kwargs): sas = SearchAndSummarize() rsp = "" info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" @@ -107,7 +87,7 @@ class RefinePRD(Refine): prompt_template, format_example = get_template(increment_template, format) prompt = prompt_template.format( - new_requirements=new_requirements, legacy=legacy, search_information=info, + context=context, legacy=legacy, search_information=info, format_example=format_example ) logger.debug(prompt) diff --git a/metagpt/actions/refine_project_management.py b/metagpt/actions/refine_project_management.py index dfeeb2db0..2775b1cb6 100644 --- a/metagpt/actions/refine_project_management.py +++ b/metagpt/actions/refine_project_management.py @@ -21,30 +21,23 @@ templates = { ## Format example {format_example} ----- -Role: You are a project manager; the goal is to perform incremental development based on the context and difference descriptions and the legacy. Break down tasks according to PRD/technical design, provide a Task list, and analyze task dependencies to start with the prerequisite modules. +Role: You are a project manager; the goal is to perform incremental development based on the context and the legacy. Break down tasks according to PRD/technical design, provide a Task list, and analyze task dependencies to start with the prerequisite modules. Requirements: Based on the context and the Legacy Project Management and Legacy Code, fill in the following missing information. Note that Please try your best to reuse legacy code, and all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file that need to modified. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. +Output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT]. The following is the attribute description of the JSON object. -## Difference Description: Provide as a python list, the foremost differences description for project management here based on the previous. - -## Incremental Required Python third-party packages: Provided as a python list, the requirements.txt format +## Required Python third-party packages: Provided as a python list, the requirements.txt format ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend based on the previous. -## Logic Analysis: Only files need to modified, Provided as a Python list[list[str]. If the file has no changes, the file will not be output. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first based on the previous. +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first. -## Task list: Only files need to modified, provided as Python list[str]. If the file has no changes, the file will not be output. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first - -output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, +Output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example, and only output the json inside this tag, nothing else """, "FORMAT_EXAMPLE": ''' { - "Incremental Requirements": "...", - "Difference Description": [ - "...", - ] - "Incremental Required Python third-party packages": [ + "Required Python third-party packages": [ "flask==1.1.2", "bcrypt==3.2.0" ], @@ -53,9 +46,6 @@ and only output the json inside this tag, nothing else ... description: A JSON object ... """, - "Logic Analysis": [ - ["game.py","Contains..."] - ], "Task list": [ "game.py" ] @@ -77,29 +67,16 @@ Role: You are a project manager; the goal is to perform incremental development Requirements: Based on the context and the Legacy Project Management and Legacy Code, fill in the following missing information. Note that Please try your best to reuse legacy code, and all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file that need to modified. Attention: Use '##' to split sections, not '#', and '## ' SHOULD WRITE BEFORE the code and triple quote. -## Difference Description: Provided as a python list, the foremost differences description for project management here based on the previous. - -## Incremental Required Python third-party packages: Provided as a python list, the requirements.txt format +## Required Python third-party packages: Provided as a python list, the requirements.txt format ## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend based on the previous. -## Logic Analysis: Only files need to modified, Provided as a Python list[list[str]. If the file has no changes, the file will not be output. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first based on the previous. - -## Task list: Only files need to modified, provided as Python list[str]. If the file has no changes, the file will not be output. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first +## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first. """, "FORMAT_EXAMPLE": ''' --- -## Incremental Requirements -... -## Difference Description -```python -[ - "The ...", -] -``` - -## Incremental Required Python third-party packages +## Required Python third-party packages ```python [ "flask==1.1.2", @@ -116,13 +93,6 @@ description: A JSON object ... """ ``` -## Logic Analysis -```python -[ - ["game.py", "Contains ..."], -] -``` - ## Task list ```python [ @@ -136,16 +106,16 @@ description: A JSON object ... OUTPUT_MAPPING = { # "Incremental Requirements": (str, ...), # ## Incremental Requirements: Provided as a str, the foremost incremental requirements for project management here based on the previous. - "Difference Description": (Union[List[str], str], ...), - "Incremental Required Python third-party packages": (Union[List[str], str], ...), + # "Difference Analysis": (Union[List[str], str], ...), + "Required Python third-party packages": (Union[List[str], str], ...), "Full API spec": (str, ...), - "Logic Analysis": (List[List[str]], ...), + # "Logic Analysis": (List[List[str]], ...), "Task list": (List[str], ...), } class RefineTasks(Action): - def __init__(self, name="CreateTasks", context=None, llm=None): + def __init__(self, name="RefineTasks", context=None, llm=None): super().__init__(name, context, llm) def _save(self, context, rsp): @@ -158,7 +128,7 @@ class RefineTasks(Action): # Write requirements.txt requirements_path = WORKSPACE_ROOT / ws_name / "requirements.txt" - requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Incremental Required Python third-party packages"))) + requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages"))) async def run(self, context, legacy, format=CONFIG.prompt_format): prompt_template, format_example = get_template(templates, format) diff --git a/metagpt/actions/write_code_guide.py b/metagpt/actions/write_code_guide.py index 391173bb3..3e51f0d2d 100644 --- a/metagpt/actions/write_code_guide.py +++ b/metagpt/actions/write_code_guide.py @@ -10,24 +10,21 @@ from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE -Role: You are a professional software engineer, and your main task is to conduct incremental development, proposing incremental development plans and code guideance based on context and legacy code. Existing code and logic that need to be retained must also appear in the code after incremental development, do not omit it. Ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). -ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example". +Role: You are a professional software engineer, and your main task is to conduct incremental development, proposing incremental development plans and code guideance based on context and legacy code. Existing code and logic that need to be retained must also appear in the code after incremental development, do not omit it. Ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language). Output format carefully referenced "Format example". ## Regulations Review: To make the software directly operable without further coding, follow the regulations below during incremental development: +0) Determine the scope of responsibilities of each file and what classes and methods need to be implemented. 1) Import all referenced classes. -2) Implement all methods. -3) Add necessary explanation to all methods. +2) Implement all methods. +3) Add necessary explanation to all methods. 4) Ensure there are no potential bugs. 5) Confirm that the entire project conforms to the tasks proposed by the user. 6) Review the code thoroughly, checking for errors and validating the logic to ensure seamless user interaction without compromising any specified requirements. -## Incremental Development Plan: Proposed the Minimum essential incremental development plan, based on the following context and legacy code by thinking step by step. -... +## Incremental Development Plan: Provided as a Python list containing `filename.py`. Proposed the detail and essential incremental development plan, based on the following context and legacy code by thinking and analyzing step by step. All incremental modules/functions need to be added to the corresponding code files. + +## Code Guidance: Propose the foremost guidelines that how to implement code of modification part for incremental development based on the above context, legacy code and incremental development plan. -## Code guidelines: Propose the foremost guidelines that how to implement code of modification part for incremental development based on the above context, legacy code and incremental development plan. -```python -... -''' ----- # Context {context} @@ -35,20 +32,31 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc ## Legacy Code You are tasked with conducting incremental development in the existing code based on the provided legacy code and above information. ``` -{code} +{legacy} ``` ----- ## Format example ----- -## Incremental Development Guide: -... +## Incremental Development Guide +[ + "`game.py` Contains `Game` and ...", +] -## Code Guidance: -# Implementation the ... + +## Code Guidance +### Implementation `xx` in `xxx.py` ..., else retain the original xxx.py code. ```python +## xxx.py ... -''' +``` +--- +### Implementation of the `Game` in `game.py` ..., else retain the original game.py code. +```python +## game.py +class Game: + ... +``` ----- """ @@ -57,11 +65,10 @@ class WriteCodeGuide(Action): def __init__(self, name="WriteCodeGuide", context: list[Message] = None, llm=None): super().__init__(name, context, llm) - async def run(self, context, code): - prompt = PROMPT_TEMPLATE.format(context=context, code=code) + async def run(self, context, legacy): + prompt = PROMPT_TEMPLATE.format(context=context, legacy=legacy) logger.info(f'Write Code Guide ..') code_guide = await self._aask(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) - # self._save(context, filename, code) return code_guide \ No newline at end of file diff --git a/metagpt/actions/write_code_refine.py b/metagpt/actions/write_code_refine.py index 466c30679..5a70b2c68 100644 --- a/metagpt/actions/write_code_refine.py +++ b/metagpt/actions/write_code_refine.py @@ -10,17 +10,19 @@ from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE Role: You are a professional engineer; your primary goal is to write PEP8 compliant, elegant, modular, easy-to-read, and maintainable Python 3.9 code (or any other programming language of your choice). -Requirements: You should modify the corresponding code based on the guidance. Then, output the complete code, fixing all errors according to the context. Ensure that you adhere to the specified guidelines for incremental development and modification of legacy code. +Requirements: Rewrite the complete code based on the Legacy Code so that it can be executed and avoid any potential bugs. You should modify the corresponding code based on the guidance. Output the complete code, fixing all errors according to the context. Ensure that you adhere to the specified guidelines for incremental development and modification of legacy code. ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format should be carefully referenced using the "Format example". Only output the current modified code, nothing else. In the modified code, if unchanged, you should output it, the complete code. -## Code: Only Write {filename}, Write code using triple quotes, based on the following list and context. -1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. -2. Requirement: Implement one of the following code files based on the provided context. Return the code in the specified format. Your code will be part of the entire project, so ensure it is complete, reliable, and reusable. -3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. -4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN. +## Rewrite Complete Code: Only Write one file {filename}, Write code using triple quotes, based on the following list, context, guidelines and legacy code. +1. Important: Do your best to implement ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT. +2. Implement one of the following code files based on the provided context. Return the code in the specified format. Your code will be part of the entire project, so ensure it is complete, reliable, and reusable. +3. Attention1: Implement the functions required by the current file scope of responsibility. For example, main only needs to focus on the basic functions of main.py in the legacy code and the incremental functions to be implemented. Reuse existing code as much as possible. You can import functions from other codes instead of reimplementing the function. If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE. +4. Attention2: Make modifications and additions to the legacy code in accordance with the provided guidelines and API. Ensure that the complete code is implemented without any omissions, taking into account the guidelines, context, and existing legacy code. Retain the basic function methods from the legacy code, and make sure to preserve the existing code and logic that needs to be retained throughout the incremental development process. Avoid omitting any essential components. 5. Think before writing: What should be implemented and provided in this document? 6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE. 7. Do not use public member functions that do not exist in your design. +8. The Modified Code is implemented according to the requirements, and there are no issues with the code logic. All functions in the Modified Code are fully implemented; none are omitted or incomplete. + ----- # Context {context} @@ -30,13 +32,13 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format should be carefull {guide} ----- -## Legacy Code: The Legacy Code that needs to be modified. +## Legacy Code: The Legacy Code that needs to be modified. '===' is the separator of each code file in the legacy code. Basic function methods need to be retained in {filename}. {legacy} ----- ## Format example ----- -## Modified/Added Code: {filename} +## Rewrite Complete Code: {filename} ```python # {filename} ... @@ -55,8 +57,8 @@ class WriteCodeRefine(Action): code = CodeParser.parse_code(block="", text=code_rsp) return code - async def run(self, context, code, filename, guide): - prompt = PROMPT_TEMPLATE.format(context=context, legacy=code, filename=filename, guide=guide) + async def run(self, context, legacy, filename, guide): + prompt = PROMPT_TEMPLATE.format(context=context, legacy=legacy, filename=filename, guide=guide) logger.info(f'Code refine {filename}..') code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index f87f8899d..00ab6de35 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -50,7 +50,11 @@ class Architect(Role): async def _act(self) -> Message: if self.increment: logger.info(f"{self._setting}: ready to RefineDesign") - legacy = self._rc.env.get_legacy()["legacy_design"] + legacy_dict = self._rc.env.get_legacy() + legacy_code = "" + for code_dict in legacy_dict.get("legacy_code"): + legacy_code += code_dict.get("code") + "\n===\n" + legacy = "Legacy Design:\n" + legacy_dict.get("legacy_design") + "\nLegacy Code:\n" + legacy_code response = await self._rc.todo.run(self._rc.history, legacy) else: diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 66ce12f83..de144d4ca 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -16,6 +16,7 @@ from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks, from metagpt.actions.refine_design_api import RefineDesign from metagpt.actions.refine_prd import RefinePRD from metagpt.actions.refine_project_management import RefineTasks +from metagpt.actions.rewrite_code import RewriteCode from metagpt.actions.write_code_refine import WriteCodeRefine from metagpt.actions.write_code_guide import WriteCodeGuide from metagpt.const import WORKSPACE_ROOT @@ -88,7 +89,7 @@ class Engineer(Role): self._init_actions([WriteCode, WriteCodeReview]) if self.increment: - self._init_actions([WriteCodeGuide, WriteCodeRefine, WriteCodeReview]) + self._init_actions([WriteCodeGuide, WriteCodeRefine, RewriteCode]) self._watch([RefineTasks]) else: self._watch([WriteTasks]) @@ -210,18 +211,21 @@ class Engineer(Role): code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later workspace = self.get_workspace() human_str = str(self._rc.memory.get_by_role("Human")[0]) - code = self._rc.env.get_legacy()["legacy_code"] + legacy_code = "" + code_list = self._rc.env.get_legacy().get("legacy_code") + for code_dict in code_list: + legacy_code += code_dict.get("code") + "\n===\n" - # Refine code + # Code guide context = [] - msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks]) - - for m in msg: - context.append(m.content) + # msg = self._rc.memory.get_by_actions([RefineDesign, RefineTasks]) + msg = self._rc.memory.get_by_actions([RefineTasks]) + for tasks_msg in msg: + context.append(tasks_msg.content) context_str = human_str + "\n".join(context) try: logger.info("Write Code Guide start!") - guide = await WriteCodeGuide().run(context=context_str, code=code) + guide = await WriteCodeGuide().run(context=context_str, legacy=legacy_code) msg = Message(content=guide, role=self.profile, cause_by=WriteCodeGuide) self._rc.memory.add(msg) except Exception as e: @@ -230,19 +234,18 @@ class Engineer(Role): # Write code or Code review for todo in self.todos: - msg = self._rc.memory.get_by_actions([RefineTasks]) - context_str = human_str + "\n".join([m.content for m in msg]) # WriteCodeRefine try: logger.info("Write Code Refine start!") - code = await WriteCodeRefine().run(context=context_str, code=code, filename=todo, guide=guide) + code = await WriteCodeRefine().run(context=context_str, legacy=legacy_code, filename=todo, guide=guide) except Exception as e: logger.error("Write Code Refine failed!", e) pass - # FIXME: Code review Action + # FIXME: Code review Action : 如果有 pass 在代码里面，需要重写 # if self.use_code_review: # try: - # rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) + # # 解析成guides context = guide + human_str + # rewrite_code = await RewriteCode().run(context=human_str, code=code, filename=todo, legacy=legacy_code) # code = rewrite_code # except Exception as e: # logger.error("code review failed!", e) @@ -251,48 +254,17 @@ class Engineer(Role): msg = Message(content=code, role=self.profile, cause_by=WriteCodeRefine) self._rc.memory.add(msg) - code_msg = todo + FILENAME_CODE_SEP + str(file_path) - code_msg_all.append(code_msg) - - logger.info(f"Done {self.get_workspace()} generating.") - msg = Message( - content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer" - ) - return msg - - async def _act_bug_fix(self, bug_msgs) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later - workspace = self.get_workspace() - flag = True - # legacy_codes = legacy.split('---') - for todo in self.todos: - context = [] - - for m in bug_msgs: - if m.sent_from != "Engineer": - context.append(m.content) - context.append(m.content) - context_str = "\n".join(context) - code = [m.content for m in bug_msgs if m.sent_from == "Engineer"] - code = "\n".join(code) - - # Refine code or Write code - # if self.increment and len(legacy_codes) > 0: - # code = legacy_codes.pop(0) - - # Code review - try: - rewrite_code = await WriteCodeGuide().run(context=context_str, code=code, filename=todo) - code = rewrite_code - except Exception as e: - logger.error("code review failed!", e) - pass - - # code = await WriteCode().run(context=context_str, filename=todo) - - file_path = self.write_file(workspace, todo, code) - msg = Message(content=code, role=self.profile, cause_by=WriteCode) - self._rc.memory.add(msg) + # 利用todo作为key，去更新code_list的code + legacy_code = "" + for code_dict in code_list: + if code_dict.get("filename") == todo: + code_dict["code"] = code + legacy_code += code_dict.get("code") + "\n===\n" + # 若code_list中没有todo，则新增一个code_dict + if todo not in [code_dict.get("filename") for code_dict in code_list]: + code_dict = {"filename": todo, "code": code} + code_list.append(code_dict) + print(len(code_list)) code_msg = todo + FILENAME_CODE_SEP + str(file_path) code_msg_all.append(code_msg) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index f48720430..2dc306297 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -52,7 +52,9 @@ class ProductManager(Role): async def _act(self) -> Message: if self.increment: logger.info(f"{self._setting}: ready to RefinePRD") - legacy = self._rc.env.get_legacy()["legacy_prd"] + legacy_dict = self._rc.env.get_legacy() + # legacy = "Legacy PRD:\n" + legacy_dict.get("legacy_prd") + "\nLegacy Code:\n" + legacy_dict.get("legacy_code") + legacy = legacy_dict.get("legacy_prd") response = await self._rc.todo.run(self._rc.history, legacy) else: diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 5d4820f4a..8cf3e579f 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -57,7 +57,10 @@ class ProjectManager(Role): human_str = "\n".join([msg.content for msg in self._rc.memory.get_by_role("Human")]) # legacy_project_management and legacy_code legacy_dict = self._rc.env.get_legacy() - legacy_str = "Legacy Project Management:\n" + legacy_dict["legacy_project_management"] + "\nLegacy Code:\n" + legacy_dict["legacy_code"] + legacy_code = "" + for code_dict in legacy_dict.get("legacy_code"): + legacy_code += code_dict.get("code") + "\n===\n" + legacy_str = "Legacy Project Management:\n" + legacy_dict["legacy_project_management"] + "\nLegacy Code:\n" + legacy_code response = await self._rc.todo.run(self._rc.history, legacy=legacy_str) else: diff --git a/metagpt/team.py b/metagpt/team.py index ea5de4a12..d71880db9 100644 --- a/metagpt/team.py +++ b/metagpt/team.py @@ -5,6 +5,8 @@ @Author : alexanderwu @File : software_company.py """ +import os + from pydantic import BaseModel, Field from metagpt.actions import BossRequirement @@ -60,3 +62,31 @@ class Team(BaseModel): await self.environment.run() return self.environment.history + def save_legacy(self, project_path): + prd_path = os.path.join(project_path, 'docs/prd.md') + design_path = os.path.join(project_path, 'docs/system_design.md') + api_spec_and_tasks_path = os.path.join(project_path, 'docs/api_spec_and_tasks.md') + code_path = os.path.join(project_path, os.path.basename(project_path)) + with open(prd_path, 'r', encoding='utf-8') as f: + legacy_prd = f.read() + with open(design_path, 'r', encoding='utf-8') as f: + legacy_design = f.read() + with open(api_spec_and_tasks_path, 'r', encoding='utf-8') as f: + legacy_project_management = f.read() + legacy_codes = [] + for root, dirs, files in os.walk(code_path): + for file in files: + legacy_code = {} + if file.endswith('.py'): + with open(os.path.join(root, file), 'r', encoding='utf-8') as f: + legacy_code['filename'] = file + legacy_code['code'] = f.read() + legacy_codes.append(legacy_code) + legacy_dict = { + 'legacy_prd': legacy_prd, + 'legacy_design': legacy_design, + 'legacy_project_management': legacy_project_management, + 'legacy_code': legacy_codes + } + self.environment.set_legacy(legacy_dict) + diff --git a/startup.py b/startup.py index 60298ed36..1ecef65b0 100644 --- a/startup.py +++ b/startup.py @@ -20,48 +20,20 @@ from metagpt.utils.special_tokens import MSG_SEP async def startup( idea: str, - difference_description: str = "", - project_path: str = "", investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False, implement: bool = True, increment: bool = False, + path: str = "", + difference: str = "", ): """Run a startup. Be a boss.""" company = Team() if increment: - # 读取文件 - prd_path = os.path.join(project_path, 'docs/prd.md') - design_path = os.path.join(project_path, 'docs/system_design.md') - api_spec_and_tasks_path = os.path.join(project_path, 'docs/api_spec_and_tasks.md') - code_path = os.path.join(project_path, os.path.basename(project_path)) - - with open(prd_path, 'r', encoding='utf-8') as f: - legacy_prd = f.read() - - with open(design_path, 'r', encoding='utf-8') as f: - legacy_design = f.read() - - with open(api_spec_and_tasks_path, 'r', encoding='utf-8') as f: - legacy_project_management = f.read() - - legacy_code = '' - for root, dirs, files in os.walk(code_path): - for file in files: - if file.endswith('.py'): - with open(os.path.join(root, file), 'r', encoding='utf-8') as f: - legacy_code += f.read() + '\n---\n' - - legacy_dict = { - 'legacy_prd': legacy_prd, - 'legacy_design': legacy_design, - 'legacy_project_management': legacy_project_management, - 'legacy_code': legacy_code - } - company.environment.set_legacy(legacy_dict) + company.save_legacy(path) company.hire( [ ProductManager(increment=increment), @@ -85,20 +57,20 @@ async def startup( company.hire([QaEngineer()]) company.invest(investment) - company.start_project(idea+"\n"+difference_description) + company.start_project(idea +"\n" + difference) await company.run(n_round=n_round) def main( idea: str, - difference_description: str = "", - project_path: str = "", investment: float = 3.0, n_round: int = 5, code_review: bool = True, run_tests: bool = False, implement: bool = True, increment: bool = False, + path: str = "", + difference: str = "", ): """ We are a software startup comprised of AI. By investing in us, @@ -111,7 +83,7 @@ def main( :return: """ asyncio.run( - startup(idea, difference_description, project_path, investment, n_round, code_review, run_tests, implement, increment)) + startup(idea, investment, n_round, code_review, run_tests, implement, increment, path, difference)) if __name__ == "__main__":