apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-06-08 15:05:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

json/markdown format

Browse source
This commit is contained in:
femto 2023-09-19 21:31:27 +08:00
parent cf05287cff
commit 7ec77e0ad7
8 changed files with 360 additions and 77 deletions
Download patch file Download diff file Expand all files Collapse all files

4
config/config.yaml
View file

@ -82,4 +82,6 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k
# MERMAID_ENGINE: nodejs
### browser path for pyppeteer engine, support Chrome, Chromium,MS Edge
#PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable"
#PYPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable"
PROMPT_FORMAT: json #json or markdown

50
metagpt/actions/action.py
View file

@ -12,6 +12,7 @@ from typing import Optional
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action_output import ActionOutput
from metagpt.config import CONFIG
from metagpt.llm import LLM
from metagpt.logs import logger
from metagpt.utils.common import OutputParser
@ -51,7 +52,12 @@ class Action(ABC):
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def _aask_v1(
self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None
self,
prompt: str,
output_class_name: str,
output_data_mapping: dict,
system_msgs: Optional[list[str]] = None,
format=CONFIG.prompt_format,
) -> ActionOutput:
"""Append default prefix"""
if not system_msgs:
@ -60,37 +66,25 @@ class Action(ABC):
content = await self.llm.aask(prompt, system_msgs)
logger.debug(content)
output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping)
parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
if format == "json":
pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]"
matches = re.findall(pattern, content, re.DOTALL)
for match in matches:
if match:
content = match
break
parsed_data = CustomDecoder(strict=False).decode(content)
else: # using markdown parser
parsed_data = OutputParser.parse_data_with_mapping(content, output_data_mapping)
logger.debug(parsed_data)
instruct_content = output_class(**parsed_data)
return ActionOutput(content, instruct_content)
@retry(stop=stop_after_attempt(3), wait=wait_fixed(1))
async def _aask_json_v1(
self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None
) -> ActionOutput:
"""Append default prefix"""
if not system_msgs:
system_msgs = []
system_msgs.append(self.prefix)
content = await self.llm.aask(prompt, system_msgs)
logger.debug(content)
output_class = ActionOutput.create_model_class(output_class_name, output_data_mapping)
pattern = r"\[CONTENT\](\s*\{.*?\}\s*)\[/CONTENT\]"
matches = re.findall(pattern, content, re.DOTALL)
extracted_content = None
for match in matches:
if match:
extracted_content = match
break
parsed_data = CustomDecoder(strict=False).decode(extracted_content)
logger.debug(parsed_data)
instruct_content = output_class(**parsed_data)
return ActionOutput(extracted_content, instruct_content)
async def run(self, *args, **kwargs):
"""Run action"""
raise NotImplementedError("The run method should be implemented in a subclass.")

85
metagpt/actions/design_api.py
View file

@ -13,10 +13,13 @@ from metagpt.actions import Action, ActionOutput
from metagpt.const import WORKSPACE_ROOT
from metagpt.logs import logger
from metagpt.utils.common import CodeParser
from metagpt.utils.get_template import get_template
from metagpt.utils.json_to_markdown import json_to_markdown
from metagpt.utils.mermaid import mermaid_to_file
PROMPT_TEMPLATE = """
templates = {
"json": {
"PROMPT_TEMPLATE": """
# Context
{context}
@ -41,8 +44,8 @@ Max Output: 8192 chars or 2048 tokens. Try to use them up.
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
and only output the json inside this tag, nothing else
"""
FORMAT_EXAMPLE = """
""",
"FORMAT_EXAMPLE": """
[CONTENT]
{
"Implementation approach": "We will ...",
@ -65,7 +68,76 @@ FORMAT_EXAMPLE = """
"Anything UNCLEAR": "The requirement is clear to me."
}
[/CONTENT]
"""
""",
},
"markdown": {
"PROMPT_TEMPLATE": """
# Context
{context}
## Format example
{format_example}
-----
Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools
Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately
Max Output: 8192 chars or 2048 tokens. Try to use them up.
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores
## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here
## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.
## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
""",
"FORMAT_EXAMPLE": """
---
## Implementation approach
We will ...
## Python package name
```python
"snake_game"
```
## File list
```python
[
"main.py",
]
```
## Data structures and interface definitions
```mermaid
classDiagram
class Game{
+int score
}
...
Game "1" -- "1" Food: has
```
## Program call flow
```mermaid
sequenceDiagram
participant M as Main
...
G->>M: end game
```
## Anything UNCLEAR
The requirement is clear to me.
---
""",
},
}
OUTPUT_MAPPING = {
"Implementation approach": (str, ...),
"Python package name": (str, ...),
@ -130,8 +202,9 @@ class WriteDesign(Action):
await self._save_system_design(docs_path, resources_path, system_design)
async def run(self, context):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
prompt_template, format_example = get_template(templates)
prompt = prompt_template.format(context=context, format_example=format_example)
# system_design = await self._aask(prompt)
system_design = await self._aask_json_v1(prompt, "system_design", OUTPUT_MAPPING)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING)
await self._save(context, system_design)
return system_design

99
metagpt/actions/project_management.py
View file

@ -10,9 +10,12 @@ from typing import List
from metagpt.actions.action import Action
from metagpt.const import WORKSPACE_ROOT
from metagpt.utils.common import CodeParser
from metagpt.utils.get_template import get_template
from metagpt.utils.json_to_markdown import json_to_markdown
PROMPT_TEMPLATE = """
templates = {
"json": {
"PROMPT_TEMPLATE": """
# Context
{context}
@ -29,7 +32,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
## Logic Analysis: Provided as a Python list[str, str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first
@ -39,9 +42,8 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
and only output the json inside this tag, nothing else
"""
FORMAT_EXAMPLE = '''
""",
"FORMAT_EXAMPLE": '''
{
"Required Python third-party packages": [
"flask==1.1.2",
@ -66,8 +68,88 @@ FORMAT_EXAMPLE = '''
""",
"Anything UNCLEAR": "We need ... how to start."
}
'''
''',
},
"markdown": {
"PROMPT_TEMPLATE": """
# Context
{context}
## Format example
{format_example}
-----
Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules
Requirements: Based on the context, fill in the following missing information, note that all sections are returned in Python code triple quote form seperatedly. Here the granularity of the task is a file, if there are any missing files, you can supplement them
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
## Required Python third-party packages: Provided in requirements.txt format
## Required Other language third-party packages: Provided in requirements.txt format
## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first
## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first.
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
""",
"FORMAT_EXAMPLE": '''
---
## Required Python third-party packages
```python
"""
flask==1.1.2
bcrypt==3.2.0
"""
```
## Required Other language third-party packages
```python
"""
No third-party ...
"""
```
## Full API spec
```python
"""
openapi: 3.0.0
...
description: A JSON object ...
"""
```
## Logic Analysis
```python
[
["game.py", "Contains ..."],
]
```
## Task list
```python
[
"game.py",
]
```
## Shared Knowledge
```python
"""
'game.py' contains ...
"""
```
## Anything UNCLEAR
We need ... how to start.
---
''',
},
}
OUTPUT_MAPPING = {
"Required Python third-party packages": (List[str], ...),
"Required Other language third-party packages": (List[str], ...),
@ -96,8 +178,9 @@ class WriteTasks(Action):
requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages")))
async def run(self, context):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
rsp = await self._aask_json_v1(prompt, "task", OUTPUT_MAPPING)
prompt_template, format_example = get_template(templates)
prompt = prompt_template.format(context=context, format_example=format_example)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING)
self._save(context, rsp)
return rsp

127
metagpt/actions/write_prd.py
View file

@ -10,8 +10,11 @@ from typing import List
from metagpt.actions import Action, ActionOutput
from metagpt.actions.search_and_summarize import SearchAndSummarize
from metagpt.logs import logger
from metagpt.utils.get_template import get_template
PROMPT_TEMPLATE = """
templates = {
"json": {
"PROMPT_TEMPLATE": """
# Context
## Original Requirements
{requirements}
@ -56,15 +59,15 @@ Requirements: According to the context, fill in the following missing informatio
## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery.
## Requirement Pool: Provided as Python list[str, str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
and only output the json inside this tag, nothing else
"""
FORMAT_EXAMPLE = """
""",
"FORMAT_EXAMPLE": """
[CONTENT]
{
"Original Requirements": "",
@ -93,7 +96,114 @@ FORMAT_EXAMPLE = """
"Anything UNCLEAR": "",
}
[/CONTENT]
"""
""",
},
"markdown": {
"PROMPT_TEMPLATE": """
# Context
## Original Requirements
{requirements}
## Search Information
{search_information}
## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the <Campain X> with REAL COMPETITOR NAME
```mermaid
quadrantChart
title Reach and engagement of campaigns
x-axis Low Reach --> High Reach
y-axis Low Engagement --> High Engagement
quadrant-1 We should expand
quadrant-2 Need to promote
quadrant-3 Re-evaluate
quadrant-4 May be improved
"Campaign: A": [0.3, 0.6]
"Campaign B": [0.45, 0.23]
"Campaign C": [0.57, 0.69]
"Campaign D": [0.78, 0.34]
"Campaign E": [0.40, 0.34]
"Campaign F": [0.35, 0.78]
"Our Target Product": [0.5, 0.6]
```
## Format example
{format_example}
-----
Role: You are a professional product manager; the goal is to design a concise, usable, efficient product
Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design
ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format.
## Original Requirements: Provide as Plain text, place the polished complete original requirements here
## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple
## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less
## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible
## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible.
## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery.
## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
""",
"FORMAT_EXAMPLE": """
---
## Original Requirements
The boss ...
## Product Goals
```python
[
"Create a ...",
]
```
## User Stories
```python
[
"As a user, ...",
]
```
## Competitive Analysis
```python
[
"Python Snake Game: ...",
]
```
## Competitive Quadrant Chart
```mermaid
quadrantChart
title Reach and engagement of campaigns
...
"Our Target Product": [0.6, 0.7]
```
## Requirement Analysis
The product should be a ...
## Requirement Pool
```python
[
["End game ...", "P0"]
]
```
## UI Design draft
Give a basic function description, and a draft
## Anything UNCLEAR
There are no unclear points.
---
""",
},
}
OUTPUT_MAPPING = {
"Original Requirements": (str, ...),
"Product Goals": (List[str], ...),
@ -120,10 +230,11 @@ class WritePRD(Action):
logger.info(sas.result)
logger.info(rsp)
prompt = PROMPT_TEMPLATE.format(
requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE
prompt_template, format_example = get_template(templates)
prompt = prompt_template.format(
requirements=requirements, search_information=info, format_example=format_example
)
logger.debug(prompt)
# prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
prd = await self._aask_json_v1(prompt, "prd", OUTPUT_MAPPING)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
return prd

10
metagpt/config.py
View file

@ -59,7 +59,7 @@ class Config(metaclass=Singleton):
self.openai_api_rpm = self._get("RPM", 3)
self.openai_api_model = self._get("OPENAI_API_MODEL", "gpt-4")
self.max_tokens_rsp = self._get("MAX_TOKENS", 2048)
self.deployment_name = self._get('DEPLOYMENT_NAME')
self.deployment_name = self._get("DEPLOYMENT_NAME")
self.deployment_id = self._get("DEPLOYMENT_ID")
self.claude_api_key = self._get("Anthropic_API_KEY")
@ -83,8 +83,10 @@ class Config(metaclass=Singleton):
self.calc_usage = self._get("CALC_USAGE", True)
self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY")
self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT")
self.mermaid_engine = self._get("MERMAID_ENGINE", 'nodejs')
self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", '')
self.mermaid_engine = self._get("MERMAID_ENGINE", "nodejs")
self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "")
self.prompt_format = self._get("PROMPT_FORMAT", "json")
def _init_with_config_files_and_env(self, configs: dict, yaml_file):
"""Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority"""
@ -113,4 +115,4 @@ class Config(metaclass=Singleton):
return value
CONFIG = Config()
CONFIG = Config()

42
metagpt/utils/common.py
View file

@ -17,20 +17,19 @@ from metagpt.logs import logger
def check_cmd_exists(command) -> int:
""" 检查命令是否存在
"""检查命令是否存在
:param command: 待检查的命令
:return: 如果命令存在，返回0，如果不存在，返回非0
"""
if platform.system().lower() == 'windows':
check_command = 'where ' + command
if platform.system().lower() == "windows":
check_command = "where " + command
else:
check_command = 'command -v ' + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }'
check_command = "command -v " + command + ' >/dev/null 2>&1 || { echo >&2 "no mermaid"; exit 1; }'
result = os.system(check_command)
return result
class OutputParser:
@classmethod
def parse_blocks(cls, text: str):
# 首先根据"##"将文本分割成不同的block
@ -54,7 +53,7 @@ class OutputParser:
@classmethod
def parse_code(cls, text: str, lang: str = "") -> str:
pattern = rf'```{lang}.*?\s+(.*?)```'
pattern = rf"```{lang}.*?\s+(.*?)```"
match = re.search(pattern, text, re.DOTALL)
if match:
code = match.group(1)
@ -65,13 +64,13 @@ class OutputParser:
@classmethod
def parse_str(cls, text: str):
text = text.split("=")[-1]
text = text.strip().strip("'").strip("\"")
text = text.strip().strip("'").strip('"')
return text
@classmethod
def parse_file_list(cls, text: str) -> list[str]:
# Regular expression pattern to find the tasks list.
pattern = r'\s*(.*=.*)?(\[.*\])'
pattern = r"\s*(.*=.*)?(\[.*\])"
# Extract tasks list string using regex.
match = re.search(pattern, text, re.DOTALL)
@ -83,12 +82,12 @@ class OutputParser:
else:
tasks = text.split("\n")
return tasks
@staticmethod
def parse_python_code(text: str) -> str:
for pattern in (
r'(.*?```python.*?\s+)?(?P<code>.*)(```.*?)',
r'(.*?```python.*?\s+)?(?P<code>.*)',
r"(.*?```python.*?\s+)?(?P<code>.*)(```.*?)",
r"(.*?```python.*?\s+)?(?P<code>.*)",
):
match = re.search(pattern, text, re.DOTALL)
if not match:
@ -135,7 +134,7 @@ class OutputParser:
typing = typing_define[0]
else:
typing = typing_define
if typing == List[str] or typing == List[Tuple[str, str]]:
if typing == List[str] or typing == List[Tuple[str, str]] or typing == List[List[str]]:
# 尝试解析list
try:
content = cls.parse_file_list(text=content)
@ -153,7 +152,6 @@ class OutputParser:
class CodeParser:
@classmethod
def parse_block(cls, block: str, text: str) -> str:
blocks = cls.parse_blocks(text)
@ -184,7 +182,7 @@ class CodeParser:
def parse_code(cls, block: str, text: str, lang: str = "") -> str:
if block:
text = cls.parse_block(block, text)
pattern = rf'```{lang}.*?\s+(.*?)```'
pattern = rf"```{lang}.*?\s+(.*?)```"
match = re.search(pattern, text, re.DOTALL)
if match:
code = match.group(1)
@ -199,7 +197,7 @@ class CodeParser:
def parse_str(cls, block: str, text: str, lang: str = ""):
code = cls.parse_code(block, text, lang)
code = code.split("=")[-1]
code = code.strip().strip("'").strip("\"")
code = code.strip().strip("'").strip('"')
return code
@classmethod
@ -207,7 +205,7 @@ class CodeParser:
# Regular expression pattern to find the tasks list.
code = cls.parse_code(block, text, lang)
# print(code)
pattern = r'\s*(.*=.*)?(\[.*\])'
pattern = r"\s*(.*=.*)?(\[.*\])"
# Extract tasks list string using regex.
match = re.search(pattern, code, re.DOTALL)
@ -230,7 +228,7 @@ class NoMoneyException(Exception):
super().__init__(self.message)
def __str__(self):
return f'{self.message} -> Amount required: {self.amount}'
return f"{self.message} -> Amount required: {self.amount}"
def print_members(module, indent=0):
@ -240,19 +238,19 @@ def print_members(module, indent=0):
:param indent:
:return:
"""
prefix = ' ' * indent
prefix = " " * indent
for name, obj in inspect.getmembers(module):
print(name, obj)
if inspect.isclass(obj):
print(f'{prefix}Class: {name}')
print(f"{prefix}Class: {name}")
# print the methods within the class
if name in ['__class__', '__base__']:
if name in ["__class__", "__base__"]:
continue
print_members(obj, indent + 2)
elif inspect.isfunction(obj):
print(f'{prefix}Function: {name}')
print(f"{prefix}Function: {name}")
elif inspect.ismethod(obj):
print(f'{prefix}Method: {name}')
print(f"{prefix}Method: {name}")
def parse_recipient(text):

20
metagpt/utils/get_template.py Normal file
View file

@ -0,0 +1,20 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/9/19 20:39
@Author : femto Zheng
@File : get_template.py
"""
from metagpt.config import CONFIG
def get_template(templates):
selected_templates = templates.get(CONFIG.prompt_format)
if selected_templates is None:
raise ValueError(f"Can't find {CONFIG.prompt_format} in passed in templates")
# Extract the selected templates
prompt_template = selected_templates["PROMPT_TEMPLATE"]
format_example = selected_templates["FORMAT_EXAMPLE"]
return prompt_template, format_example