apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-06-23 15:48:11 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: merge geekan:cli-etc

Browse source
This commit is contained in:
莘权 马 2023-11-28 18:16:50 +08:00
commit 78548c2ddc
84 changed files with 2982 additions and 1000 deletions
Download patch file Download diff file Expand all files Collapse all files

4
metagpt/actions/__init__.py
View file

@ -9,7 +9,7 @@ from enum import Enum
from metagpt.actions.action import Action
from metagpt.actions.action_output import ActionOutput
from metagpt.actions.add_requirement import BossRequirement
from metagpt.actions.add_requirement import UserRequirement
from metagpt.actions.debug_error import DebugError
from metagpt.actions.design_api import WriteDesign
from metagpt.actions.design_api_review import DesignReview
@ -28,7 +28,7 @@ from metagpt.actions.write_test import WriteTest
class ActionType(Enum):
"""All types of Actions, used for indexing."""
ADD_REQUIREMENT = BossRequirement
ADD_REQUIREMENT = UserRequirement
WRITE_PRD = WritePRD
WRITE_PRD_REVIEW = WritePRDReview
WRITE_DESIGN = WriteDesign

4
metagpt/actions/action.py
View file

@ -30,6 +30,10 @@ class Action(ABC):
self.desc = ""
self.content = ""
self.instruct_content = None
self.env = None
def set_env(self, env):
self.env = env
def set_prefix(self, prefix, profile):
"""Set prefix for later usage"""

4
metagpt/actions/add_requirement.py
View file

@ -8,8 +8,8 @@
from metagpt.actions import Action
class BossRequirement(Action):
"""Boss Requirement without any implementation details"""
class UserRequirement(Action):
"""User Requirement without any implementation details"""
async def run(self, *args, **kwargs):
raise NotImplementedError

12
metagpt/actions/debug_error.py
View file

@ -51,7 +51,9 @@ class DebugError(Action):
super().__init__(name, context, llm)
async def run(self, *args, **kwargs) -> str:
output_doc = await FileRepository.get_file(filename=self.context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO)
output_doc = await FileRepository.get_file(
filename=self.context.output_filename, relative_path=TEST_OUTPUTS_FILE_REPO
)
if not output_doc:
return ""
output_detail = RunCodeResult.loads(output_doc.content)
@ -61,10 +63,14 @@ class DebugError(Action):
return ""
logger.info(f"Debug and rewrite {self.context.code_filename}")
code_doc = await FileRepository.get_file(filename=self.context.code_filename, relative_path=CONFIG.src_workspace)
code_doc = await FileRepository.get_file(
filename=self.context.code_filename, relative_path=CONFIG.src_workspace
)
if not code_doc:
return ""
test_doc = await FileRepository.get_file(filename=self.context.test_filename, relative_path=TEST_CODES_FILE_REPO)
test_doc = await FileRepository.get_file(
filename=self.context.test_filename, relative_path=TEST_CODES_FILE_REPO
)
if not test_doc:
return ""
prompt = PROMPT_TEMPLATE.format(code=code_doc.content, test_code=test_doc.content, logs=output_detail.stderr)

117
metagpt/actions/design_api.py
View file

@ -37,21 +37,21 @@ templates = {
## Format example
{format_example}
-----
Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools
Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
Requirement: Fill in the following missing information based on the context, each section name is a key in json
Max Output: 8192 chars or 2048 tokens. Try to use them up.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select appropriate open-source frameworks.
## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here
## File list: Provided as Python list[str], the list of files needed (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.
## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.
## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
## Anything UNCLEAR: Provide as Plain text. Try to clarify it.
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
and only output the json inside this tag, nothing else
@ -60,9 +60,9 @@ and only output the json inside this tag, nothing else
[CONTENT]
{
"Implementation approach": "We will ...",
"Python package name": "snake_game",
"project_name": "snake_game",
"File list": ["main.py"],
"Data structures and interface definitions": '
"Data structures and interfaces": '
classDiagram
class Game{
+int score
@ -90,21 +90,21 @@ and only output the json inside this tag, nothing else
{format_example}
-----
Role: You are an architect; the goal is to design a SOTA PEP8-compliant python system; make the best use of good open source tools
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
Requirement: Fill in the following missing information based on the context, note that all sections are response with code form separately
Max Output: 8192 chars or 2048 tokens. Try to use them up.
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
ATTENTION: Output carefully referenced "Format example" in format.
## Implementation approach: Provide as Plain text. Analyze the difficult points of the requirements, select the appropriate open-source framework.
## Python package name: Provide as Python str with python triple quoto, concise and clear, characters only use a combination of all lowercase and underscores
## project_name: Provide as Plain text, concise and clear, characters only use a combination of all lowercase and underscores
## File list: Provided as Python list[str], the list of ONLY REQUIRED files needed to write the program(LESS IS MORE!). Only need relative paths, comply with PEP8 standards. ALWAYS write a main.py or app.py here
## File list: Provided as Python list[str], the list of code files (including HTML & CSS IF NEEDED) to write the program. Only need relative paths. ALWAYS write a main.py or app.py here
## Data structures and interface definitions: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.
## Data structures and interfaces: Use mermaid classDiagram code syntax, including classes (INCLUDING __init__ method) and functions (with type annotations), CLEARLY MARK the RELATIONSHIPS between classes, and comply with PEP8 standards. The data structures SHOULD BE VERY DETAILED and the API should be comprehensive with a complete design.
## Program call flow: Use sequenceDiagram code syntax, COMPLETE and VERY DETAILED, using CLASSES AND API DEFINED ABOVE accurately, covering the CRUD AND INIT of each object, SYNTAX MUST BE CORRECT.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
## Anything UNCLEAR: Provide as Plain text. Try to clarify it.
""",
"FORMAT_EXAMPLE": """
@ -112,7 +112,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Implementation approach
We will ...
## Python package name
## project_name
```python
"snake_game"
```
@ -124,7 +124,7 @@ We will ...
]
```
## Data structures and interface definitions
## Data structures and interfaces
```mermaid
classDiagram
class Game{
@ -151,9 +151,9 @@ The requirement is clear to me.
OUTPUT_MAPPING = {
"Implementation approach": (str, ...),
"Python package name": (str, ...),
"project_name": (str, ...),
"File list": (List[str], ...),
"Data structures and interface definitions": (str, ...),
"Data structures and interfaces": (str, ...),
"Program call flow": (str, ...),
"Anything UNCLEAR": (str, ...),
}
@ -226,19 +226,76 @@ class WriteDesign(Action):
# leaving room for global optimization in subsequent steps.
return ActionOutput(content=changed_files.json(), instruct_content=changed_files)
# =======
# def recreate_workspace(self, workspace: Path):
# try:
# shutil.rmtree(workspace)
# except FileNotFoundError:
# pass # Folder does not exist, but we don't care
# workspace.mkdir(parents=True, exist_ok=True)
# async def _save_prd(self, docs_path, resources_path, context):
# prd_file = docs_path / "prd.md"
# if context[-1].instruct_content and context[-1].instruct_content.dict()["Competitive Quadrant Chart"]:
# quadrant_chart = context[-1].instruct_content.dict()["Competitive Quadrant Chart"]
# await mermaid_to_file(quadrant_chart, resources_path / "competitive_analysis")
#
# if context[-1].instruct_content:
# logger.info(f"Saving PRD to {prd_file}")
# prd_file.write_text(context[-1].instruct_content.json(ensure_ascii=False), encoding='utf-8')
# async def _save_system_design(self, docs_path, resources_path, system_design):
# data_api_design = system_design.instruct_content.dict()[
# "Data structures and interfaces"
# ] # CodeParser.parse_code(block="Data structures and interfaces", text=content)
# seq_flow = system_design.instruct_content.dict()[
# "Program call flow"
# ] # CodeParser.parse_code(block="Program call flow", text=content)
# await mermaid_to_file(data_api_design, resources_path / "data_api_design")
# await mermaid_to_file(seq_flow, resources_path / "seq_flow")
# system_design_file = docs_path / "system_design.md"
# logger.info(f"Saving System Designs to {system_design_file}")
# system_design_file.write_text(system_design.instruct_content.json(ensure_ascii=False), encoding='utf-8')
# async def _save(self, context, system_design):
# if isinstance(system_design, ActionOutput):
# project_name = system_design.instruct_content.dict()["project_name"]
# else:
# project_name = CodeParser.parse_str(block="project_name", text=system_design)
# workspace = CONFIG.workspace_path / project_name
# self.recreate_workspace(workspace)
# docs_path = workspace / "docs"
# resources_path = workspace / "resources"
# docs_path.mkdir(parents=True, exist_ok=True)
# resources_path.mkdir(parents=True, exist_ok=True)
# await self._save_prd(docs_path, resources_path, context)
# await self._save_system_design(docs_path, resources_path, system_design)
# async def run(self, context, format=CONFIG.prompt_format):
async def _new_system_design(self, context, format=CONFIG.prompt_format):
prompt_template, format_example = get_template(templates, format)
prompt = prompt_template.format(context=context, format_example=format_example)
# system_design = await self._aask(prompt)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
# fix Python package name, we can't system_design.instruct_content.python_package_name = "xxx" since "Python
# package name" contain space, have to use setattr
# fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name"
# contain space, have to use setattr
setattr(
system_design.instruct_content,
"Python package name",
system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'),
"project_name",
system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'),
)
await self._rename_workspace(system_design)
# =======
# # fix project_name, we can't system_design.instruct_content.python_package_name = "xxx" since "project_name" contain space, have to use setattr
# # setattr(
# # system_design.instruct_content,
# # "project_name",
# # system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'),
# # )
# await self._save(context, system_design)
# >>>>>>> feature/geekan_cli_etc
return system_design
async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):
@ -248,10 +305,10 @@ class WriteDesign(Action):
# package name" contain space, have to use setattr
setattr(
system_design.instruct_content,
"Python package name",
system_design.instruct_content.dict()["Python package name"].strip().strip("'").strip('"'),
"project_name",
system_design.instruct_content.dict()["project_name"].strip().strip("'").strip('"'),
)
system_design_doc.content = system_design.instruct_content.json()
system_design_doc.content = system_design.instruct_content.json(ensure_ascii=False)
return system_design_doc
@staticmethod
@ -260,9 +317,9 @@ class WriteDesign(Action):
return
if isinstance(system_design, ActionOutput):
ws_name = system_design.instruct_content.dict()["Python package name"]
ws_name = system_design.instruct_content.dict()["project_name"]
else:
ws_name = CodeParser.parse_str(block="Python package name", text=system_design)
ws_name = CodeParser.parse_str(block="project_name", text=system_design)
CONFIG.git_repo.rename_root(ws_name)
async def _update_system_design(self, filename, prds_file_repo, system_design_file_repo) -> Document:
@ -271,7 +328,9 @@ class WriteDesign(Action):
if not old_system_design_doc:
system_design = await self._new_system_design(context=prd.content)
doc = Document(
root_path=SYSTEM_DESIGN_FILE_REPO, filename=filename, content=system_design.instruct_content.json()
root_path=SYSTEM_DESIGN_FILE_REPO,
filename=filename,
content=system_design.instruct_content.json(ensure_ascii=False),
)
else:
doc = await self._merge(prd_doc=prd, system_design_doc=old_system_design_doc)

65
metagpt/actions/project_management.py
View file

@ -15,7 +15,12 @@ from typing import List
from metagpt.actions import ActionOutput
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO, TASK_PDF_FILE_REPO, PACKAGE_REQUIREMENTS_FILENAME
from metagpt.const import (
PACKAGE_REQUIREMENTS_FILENAME,
SYSTEM_DESIGN_FILE_REPO,
TASK_FILE_REPO,
TASK_PDF_FILE_REPO,
)
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.file_repository import FileRepository
@ -31,22 +36,23 @@ templates = {
{format_example}
-----
Role: You are a project manager; the goal is to break down tasks according to PRD/technical design, give a task list, and analyze task dependencies to start with the prerequisite modules
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
Requirements: Based on the context, fill in the following missing information, each section name is a key in json. Here the granularity of the task is a file, if there are any missing files, you can supplement them
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote.
ATTENTION: Output carefully referenced "Format example" in format.
## Required Python third-party packages: Provided in requirements.txt format
## Required Python third-party packages: Provide Python list[str] in requirements.txt format
## Required Other language third-party packages: Provided in requirements.txt format
## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
## Required Other language third-party packages: Provide Python list[str] in requirements.txt format
## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first
## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first.
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs.
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
and only output the json inside this tag, nothing else
@ -60,17 +66,17 @@ and only output the json inside this tag, nothing else
"Required Other language third-party packages": [
"No third-party ..."
],
"Logic Analysis": [
["game.py", "Contains..."]
],
"Task list": [
"game.py"
],
"Full API spec": """
openapi: 3.0.0
...
description: A JSON object ...
""",
"Logic Analysis": [
["game.py","Contains..."]
],
"Task list": [
"game.py"
],
"Shared Knowledge": """
'game.py' contains ...
""",
@ -94,15 +100,15 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Required Other language third-party packages: Provided in requirements.txt format
## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
## Logic Analysis: Provided as a Python list[list[str]. the first is filename, the second is class/method/function should be implemented in this file. Analyze the dependencies between the files, which work should be done first
## Task list: Provided as Python list[str]. Each str is a filename, the more at the beginning, the more it is a prerequisite dependency, should be done first
## Full API spec: Use OpenAPI 3.0. Describe all APIs that may be used by both frontend and backend.
## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first.
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs.
""",
"FORMAT_EXAMPLE": '''
@ -134,14 +140,16 @@ description: A JSON object ...
## Logic Analysis
```python
[
["game.py", "Contains ..."],
["index.js", "Contains ..."],
["main.py", "Contains ..."],
]
```
## Task list
```python
[
"game.py",
"index.js",
"main.py",
]
```
@ -239,7 +247,9 @@ class WriteTasks(Action):
task_doc = await self._merge(system_design_doc=system_design_doc, task_doc=task_doc)
else:
rsp = await self._run_new_tasks(context=system_design_doc.content)
task_doc = Document(root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json())
task_doc = Document(
root_path=TASK_FILE_REPO, filename=filename, content=rsp.instruct_content.json(ensure_ascii=False)
)
await tasks_file_repo.save(
filename=filename, content=task_doc.content, dependencies={system_design_doc.root_relative_path}
)
@ -248,6 +258,21 @@ class WriteTasks(Action):
return task_doc
async def _run_new_tasks(self, context, format=CONFIG.prompt_format):
# =======
# def _save(self, context, rsp):
# if context[-1].instruct_content:
# ws_name = context[-1].instruct_content.dict()["project_name"]
# else:
# ws_name = CodeParser.parse_str(block="project_name", text=context[-1].content)
# file_path = CONFIG.workspace_path / ws_name / "docs/api_spec_and_tasks.md"
# file_path.write_text(rsp.instruct_content.json(ensure_ascii=False))
#
# # Write requirements.txt
# requirements_path = CONFIG.workspace_path / ws_name / "requirements.txt"
# requirements_path.write_text("\n".join(rsp.instruct_content.dict().get("Required Python third-party packages")))
#
# async def run(self, context, format=CONFIG.prompt_format):
# >>>>>>> feature/geekan_cli_etc
prompt_template, format_example = get_template(templates, format)
prompt = prompt_template.format(context=context, format_example=format_example)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
@ -256,7 +281,7 @@ class WriteTasks(Action):
async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document:
prompt = MERGE_PROMPT.format(context=system_design_doc.content, old_tasks=task_doc.content)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
task_doc.content = rsp.instruct_content.json()
task_doc.content = rsp.instruct_content.json(ensure_ascii=False)
return task_doc
@staticmethod

115
metagpt/actions/summarize_code.py Normal file
View file

@ -0,0 +1,115 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Author : alexanderwu
@File : summarize_code.py
"""
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.utils.file_repository import FileRepository
PROMPT_TEMPLATE = """
NOTICE
Role: You are a professional software engineer, and your main task is to review the code.
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example".
-----
# System Design
```text
{system_design}
```
-----
# Tasks
```text
{tasks}
```
-----
{code_blocks}
## Code Review All: 请你对历史所有文件进行阅读，在文件中找到可能的bug，如函数未实现、调用错误、未引用等
## Call flow: mermaid代码，根据实现的函数，使用mermaid绘制完整的调用链
## Summary: 根据历史文件的实现情况进行总结
## TODOs: Python dict[str, str]，这里写出需要修改的文件列表与理由，我们会在之后进行修改
"""
FORMAT_EXAMPLE = """
## Code Review All
### a.py
- 它少实现了xxx需求...
- 字段yyy没有给出...
- ...
### b.py
...
### c.py
...
## Call flow
```mermaid
flowchart TB
c1-->a2
subgraph one
a1-->a2
end
subgraph two
b1-->b2
end
subgraph three
c1-->c2
end
```
## Summary
- a.py:...
- b.py:...
- c.py:...
- ...
## TODOs
{
"a.py": "implement requirement xxx...",
}
"""
class SummarizeCode(Action):
def __init__(self, name="SummarizeCode", context=None, llm=None):
super().__init__(name, context, llm)
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def summarize_code(self, prompt):
code_rsp = await self._aask(prompt)
return code_rsp
async def run(self):
design_doc = await FileRepository.get_file(self.context.design_filename)
task_doc = await FileRepository.get_file(self.context.task_filename)
src_file_repo = CONFIG.git_repo.new_file_repository(relative_path=CONFIG.src_workspace)
code_blocks = []
for filename in self.context.codes_filenames:
code_doc = await src_file_repo.get(filename)
code_block = f"```python\n{code_doc.content}\n```\n-----"
code_blocks.append(code_block)
format_example = FORMAT_EXAMPLE
prompt = PROMPT_TEMPLATE.format(
system_design=design_doc.content,
tasks=task_doc.content,
code_blocks="\n".join(code_blocks),
format_example=format_example,
)
logger.info("Summarize code..")
rsp = await self.summarize_code(prompt)
return rsp

42
metagpt/actions/write_code.py
View file

@ -15,10 +15,10 @@
RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
"""
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import TEST_OUTPUTS_FILE_REPO
from metagpt.logs import logger
from metagpt.schema import CodingContext, RunCodeResult
@ -28,16 +28,24 @@ from metagpt.utils.file_repository import FileRepository
PROMPT_TEMPLATE = """
NOTICE
Role: You are a professional engineer; the main goal is to write PEP8 compliant, elegant, modular, easy to read and maintain Python 3.9 code (but you can also use other programming language)
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example".
-----
# Context
{context}
-----
## Code: {filename} Write code with triple quoto, based on the following list and context.
1. Do your best to implement THIS ONLY ONE FILE. ONLY USE EXISTING API. IF NO API, IMPLEMENT IT.
2. Requirement: Based on the context, implement one following code file, note to return only in code form, your code will be part of the entire project, so please implement complete, reliable, reusable code snippets
3. Attention1: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE.
4. Attention2: YOU MUST FOLLOW "Data structures and interface definitions". DONT CHANGE ANY DESIGN.
3. Set default value: If there is any setting, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE.
4. Follow design: YOU MUST FOLLOW "Data structures and interfaces". DONT CHANGE ANY DESIGN.
5. Think before writing: What should be implemented and provided in this document?
6. CAREFULLY CHECK THAT YOU DONT MISS ANY NECESSARY CLASS/FUNCTION IN THIS FILE.
7. Do not use public member functions that do not exist in your design.
8. Before using a variable, make sure you reference it first
9. Write out EVERY DETAIL, DON'T LEAVE TODO.
-----
# Design
@ -75,6 +83,29 @@ class WriteCode(Action):
def __init__(self, name="WriteCode", context=None, llm=None):
super().__init__(name, context, llm)
# <<<<<<< HEAD
# =======
# def _is_invalid(self, filename):
# return any(i in filename for i in ["mp3", "wav"])
#
# def _save(self, context, filename, code):
# # logger.info(filename)
# # logger.info(code_rsp)
# if self._is_invalid(filename):
# return
#
# design = [i for i in context if i.cause_by == WriteDesign][0]
#
# ws_name = CodeParser.parse_str(block="project_name", text=design.content)
# ws_path = CONFIG.workspace_path / ws_name
# if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]):
# ws_path = ws_path / ws_name
# code_path = ws_path / filename
# code_path.parent.mkdir(parents=True, exist_ok=True)
# code_path.write_text(code)
# logger.info(f"Saving Code to {code_path}")
#
# >>>>>>> feature/geekan_cli_etc
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def write_code(self, prompt) -> str:
code_rsp = await self._aask(prompt)
@ -83,8 +114,9 @@ class WriteCode(Action):
async def run(self, *args, **kwargs) -> CodingContext:
coding_context = CodingContext.loads(self.context.content)
test_doc = await FileRepository.get_file(filename="test_" + coding_context.filename + ".json",
relative_path=TEST_OUTPUTS_FILE_REPO)
test_doc = await FileRepository.get_file(
filename="test_" + coding_context.filename + ".json", relative_path=TEST_OUTPUTS_FILE_REPO
)
logs = ""
if test_doc:
test_detail = RunCodeResult.loads(test_doc.content)

127
metagpt/actions/write_code_review.py
View file

@ -11,6 +11,7 @@
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.schema import CodingContext
from metagpt.utils.common import CodeParser
@ -18,49 +19,74 @@ from metagpt.utils.common import CodeParser
PROMPT_TEMPLATE = """
NOTICE
Role: You are a professional software engineer, and your main task is to review the code. You need to ensure that the code conforms to the PEP8 standards, is elegantly designed and modularized, easy to read and maintain, and is written in Python 3.9 (or in another programming language).
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenced "Format example".
## Code Review: Based on the following context and code, and following the check list, Provide key, clear, concise, and specific code modification suggestions, up to 5.
```
1. Check 0: Is the code implemented as per the requirements?
2. Check 1: Are there any issues with the code logic?
3. Check 2: Does the existing code follow the "Data structures and interface definitions"?
4. Check 3: Is there a function in the code that is omitted or not fully implemented that needs to be implemented?
5. Check 4: Does the code have unnecessary or lack dependencies?
```
## Rewrite Code: {filename} Base on "Code Review" and the source code, rewrite code with triple quotes. Do your utmost to optimize THIS SINGLE FILE.
-----
# Context
{context}
## Code: {filename}
## Code to be Reviewed: {filename}
```
{code}
```
-----
## Code Review: Based on the "Code to be Reviewed", provide key, clear, concise, and specific code modification suggestions, up to 5.
1. Is the code implemented as per the requirements? If not, how to achieve it? Analyse it step by step.
2. Is the code logic completely correct? If there are errors, please indicate how to correct them.
3. Does the existing code follow the "Data structures and interfaces"?
4. Are all functions implemented? If there is no implementation, please indicate how to achieve it step by step.
5. Have all necessary pre-dependencies been imported? If not, indicate which ones need to be imported
6. Is the code implemented concisely enough? Are methods from other files being reused correctly?
## Code Review Result: If the code doesn't have bugs, we don't need to rewrite it, so answer LGTM and stop. ONLY ANSWER LGTM/LBTM.
LGTM/LBTM
## Rewrite Code: if it still has some bugs, rewrite {filename} based on "Code Review" with triple quotes, try to get LGTM. Do your utmost to optimize THIS SINGLE FILE. Implement ALL TODO. RETURN ALL CODE, NEVER OMIT ANYTHING. 以任何方式省略代码都是不允许的。
```
```
## Format example
-----
{format_example}
-----
"""
FORMAT_EXAMPLE = """
## Code Review
1. The code ...
-----
# EXAMPLE 1
## Code Review: {filename}
1. No, we should add the logic of ...
2. ...
3. ...
4. ...
5. ...
6. ...
## Code Review Result: {filename}
LBTM
## Rewrite Code: {filename}
```python
## {filename}
...
```
-----
# EXAMPLE 2
## Code Review: {filename}
1. Yes.
2. Yes.
3. Yes.
4. Yes.
5. Yes.
6. Yes.
## Code Review Result: {filename}
LGTM
## Rewrite Code: {filename}
pass
-----
"""
@ -69,23 +95,60 @@ class WriteCodeReview(Action):
super().__init__(name, context, llm)
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def write_code(self, prompt):
async def write_code_review_and_rewrite(self, prompt):
code_rsp = await self._aask(prompt)
result = CodeParser.parse_block("Code Review Result", code_rsp)
if "LGTM" in result:
return result, None
code = CodeParser.parse_code(block="", text=code_rsp)
return code
return result, code
# <<<<<<< HEAD
# async def run(self, *args, **kwargs) -> CodingContext:
# format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename)
# context = "\n----------\n".join(
# [self.context.design_doc.content, self.context.task_doc.content, self.context.code_doc.content]
# )
# prompt = PROMPT_TEMPLATE.format(
# context=context,
# code=self.context.code_doc.content,
# filename=self.context.code_doc.filename,
# format_example=format_example,
# )
# logger.info(f"Code review {self.context.code_doc.filename}..")
# code = await self.write_code(prompt)
# self.context.code_doc.content = code
# return self.context
# =======
async def run(self, *args, **kwargs) -> CodingContext:
format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename)
context = "\n".join(
[self.context.design_doc.content, self.context.task_doc.content, self.context.code_doc.content]
)
prompt = PROMPT_TEMPLATE.format(
context=context,
code=self.context.code_doc.content,
filename=self.context.code_doc.filename,
format_example=format_example,
)
logger.info(f"Code review {self.context.code_doc.filename}..")
code = await self.write_code(prompt)
self.context.code_doc.content = code
iterative_code = self.context.code_doc.content
k = CONFIG.code_review_k_times or 1
for i in range(k):
format_example = FORMAT_EXAMPLE.format(filename=self.context.code_doc.filename)
context = "\n----------\n".join(
[
"```text\n" + self.context.design_doc.content + "```\n",
"```text\n" + self.context.task_doc.content + "```\n",
"```python\n" + self.context.code_doc.content + "```\n",
]
)
prompt = PROMPT_TEMPLATE.format(
context=context,
code=iterative_code,
filename=self.context.code_doc.filename,
format_example=format_example,
)
logger.info(
f"Code review and rewrite {self.context.code_doc.filename,}: {i+1}/{k} | {len(iterative_code)=}, {len(self.context.code_doc.content)=}"
)
result, rewrited_code = await self.write_code_review_and_rewrite(prompt)
if "LBTM" in result:
iterative_code = rewrited_code
elif "LGTM" in result:
self.context.code_doc.content = iterative_code
return self.context
# code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING)
# self._save(context, filename, code)
# 如果rewrited_code是None（原code perfect），那么直接返回code
self.context.code_doc.content = iterative_code
return self.context

105
metagpt/actions/write_prd.py
View file

@ -35,53 +35,50 @@ templates = {
"json": {
"PROMPT_TEMPLATE": """
# Context
## Original Requirements
{requirements}
## Search Information
{search_information}
## mermaid quadrantChart code syntax example. DONT USE QUOTO IN CODE DUE TO INVALID SYNTAX. Replace the <Campain X> with REAL COMPETITOR NAME
```mermaid
quadrantChart
title Reach and engagement of campaigns
x-axis Low Reach --> High Reach
y-axis Low Engagement --> High Engagement
quadrant-1 We should expand
quadrant-2 Need to promote
quadrant-3 Re-evaluate
quadrant-4 May be improved
"Campaign: A": [0.3, 0.6]
"Campaign B": [0.45, 0.23]
"Campaign C": [0.57, 0.69]
"Campaign D": [0.78, 0.34]
"Campaign E": [0.40, 0.34]
"Campaign F": [0.35, 0.78]
"Our Target Product": [0.5, 0.6]
```
{{
"Original Requirements": "{requirements}",
"Search Information": ""
}}
## Format example
{format_example}
-----
Role: You are a professional product manager; the goal is to design a concise, usable, efficient product
Requirements: According to the context, fill in the following missing information, each section name is a key in json ,If the requirements are unclear, ensure minimum viability and avoid excessive design
Language: Please use the same language as the user requirement, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly.
ATTENTION: Output carefully referenced "Format example" in format.
## Original Requirements: Provide as Plain text, place the polished complete original requirements here
## YOU NEED TO FULFILL THE BELOW JSON DOC
## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple
## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less
## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible
## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible.
## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery.
## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
"Original Requirements": "", # str, place the polished complete original requirements here
"project_name": "", # str, name it like game_2048 / web_2048 / simple_crm etc.
"Search Information": "",
"Requirements": "",
"Product Goals": [], # Provided as Python list[str], up to 3 clear, orthogonal product goals.
"User Stories": [], # Provided as Python list[str], up to 5 scenario-based user stories
"Competitive Analysis": [], # Provided as Python list[str], up to 8 competitive product analyses
# Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible.
"Competitive Quadrant Chart": "quadrantChart
title Reach and engagement of campaigns
x-axis Low Reach --> High Reach
y-axis Low Engagement --> High Engagement
quadrant-1 We should expand
quadrant-2 Need to promote
quadrant-3 Re-evaluate
quadrant-4 May be improved
Campaign A: [0.3, 0.6]
Campaign B: [0.45, 0.23]
Campaign C: [0.57, 0.69]
Campaign D: [0.78, 0.34]
Campaign E: [0.40, 0.34]
Campaign F: [0.35, 0.78]",
"Requirement Analysis": "", # Provide as Plain text.
"Requirement Pool": [["P0","P0 requirement"],["P1","P1 requirement"]], # Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards
"UI Design draft": "", # Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
"Anything UNCLEAR": "", # Provide as Plain text. Try to clarify it.
}}
output a properly formatted JSON, wrapped inside [CONTENT][/CONTENT] like format example,
and only output the json inside this tag, nothing else
@ -89,6 +86,7 @@ and only output the json inside this tag, nothing else
"FORMAT_EXAMPLE": """
[CONTENT]
{
"Language": "",
"Original Requirements": "",
"Search Information": "",
"Requirements": "",
@ -149,30 +147,33 @@ quadrantChart
{format_example}
-----
Role: You are a professional product manager; the goal is to design a concise, usable, efficient product
Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly. If the requirements are unclear, ensure minimum viability and avoid excessive design
Language: Please use the same language as the user requirement to answer, but the title and code should be still in English. For example, if the user speaks Chinese, the specific text of your answer should also be in Chinese.
Requirements: According to the context, fill in the following missing information, note that each sections are returned in Python code triple quote form seperatedly.
ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. AND '## <SECTION_NAME>' SHOULD WRITE BEFORE the code and triple quote. Output carefully referenced "Format example" in format.
## Language: Provide as Plain text, use the same language as the user requirement.
## Original Requirements: Provide as Plain text, place the polished complete original requirements here
## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals. If the requirement itself is simple, the goal should also be simple
## Product Goals: Provided as Python list[str], up to 3 clear, orthogonal product goals.
## User Stories: Provided as Python list[str], up to 5 scenario-based user stories, If the requirement itself is simple, the user stories should also be less
## User Stories: Provided as Python list[str], up to 5 scenario-based user stories
## Competitive Analysis: Provided as Python list[str], up to 7 competitive product analyses, consider as similar competitors as possible
## Competitive Quadrant Chart: Use mermaid quadrantChart code syntax. up to 14 competitive products. Translation: Distribute these competitor scores evenly between 0 and 1, trying to conform to a normal distribution centered around 0.5 as much as possible.
## Requirement Analysis: Provide as Plain text. Be simple. LESS IS MORE. Make your requirements less dumb. Delete the parts unnessasery.
## Requirement Analysis: Provide as Plain text.
## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards; no more than 5 requirements and consider to make its difficulty lower
## Requirement Pool: Provided as Python list[list[str], the parameters are requirement description, priority(P0/P1/P2), respectively, comply with PEP standards
## UI Design draft: Provide as Plain text. Be simple. Describe the elements and functions, also provide a simple style description and layout description.
## Anything UNCLEAR: Provide as Plain text. Make clear here.
## Anything UNCLEAR: Provide as Plain text. Try to clarify it.
""",
"FORMAT_EXAMPLE": """
---
## Original Requirements
The boss ...
The user ...
## Product Goals
```python
@ -225,6 +226,7 @@ There are no unclear points.
OUTPUT_MAPPING = {
"Language": (str, ...),
"Original Requirements": (str, ...),
"Product Goals": (List[str], ...),
"User Stories": (List[str], ...),
@ -322,11 +324,14 @@ class WritePRD(Action):
logger.info(sas.result)
logger.info(rsp)
# logger.info(format)
prompt_template, format_example = get_template(templates, format)
# logger.info(prompt_template)
# logger.info(format_example)
prompt = prompt_template.format(
requirements=requirements, search_information=info, format_example=format_example
)
logger.debug(prompt)
# logger.info(prompt)
# prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
return prd
@ -346,7 +351,7 @@ class WritePRD(Action):
async def _merge(self, new_requirement_doc, prd_doc, format=CONFIG.prompt_format) -> Document:
prompt = MERGE_PROMPT.format(requirements=new_requirement_doc.content, old_prd=prd_doc.content)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING, format=format)
prd_doc.content = prd.instruct_content.json()
prd_doc.content = prd.instruct_content.json(ensure_ascii=False)
return prd_doc
async def _update_prd(self, requirement_doc, prd_doc, prds_file_repo, *args, **kwargs) -> Document | None:
@ -355,7 +360,7 @@ class WritePRD(Action):
new_prd_doc = Document(
root_path=PRDS_FILE_REPO,
filename=FileRepository.new_filename() + ".json",
content=prd.instruct_content.json(),
content=prd.instruct_content.json(ensure_ascii=False),
)
elif await self._is_relative_to(requirement_doc, prd_doc):
new_prd_doc = await self._merge(requirement_doc, prd_doc)

4
metagpt/actions/write_test.py
View file

@ -3,7 +3,7 @@
"""
@Time : 2023/5/11 22:12
@Author : alexanderwu
@File : environment.py
@File : write_test.py
@Modified By: mashenquan, 2023-11-27. Following the think-act principle, solidify the task parameters when creating the
WriteTest object, rather than passing them in when calling the run function.
"""
@ -19,7 +19,7 @@ NOTICE
2. Requirement: Based on the context, develop a comprehensive test suite that adequately covers all relevant aspects of the code file under review. Your test suite will be part of the overall project QA, so please develop complete, robust, and reusable test cases.
3. Attention1: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the test case or script.
4. Attention2: If there are any settings in your tests, ALWAYS SET A DEFAULT VALUE, ALWAYS USE STRONG TYPE AND EXPLICIT VARIABLE.
5. Attention3: YOU MUST FOLLOW "Data structures and interface definitions". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity.
5. Attention3: YOU MUST FOLLOW "Data structures and interfaces". DO NOT CHANGE ANY DESIGN. Make sure your tests respect the existing design and ensure its validity.
6. Think before writing: What should be tested and validated in this document? What edge cases could exist? What might fail?
7. CAREFULLY CHECK THAT YOU DON'T MISS ANY NECESSARY TEST CASES/SCRIPTS IN THIS FILE.
Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD WRITE BEFORE the test case or script and triple quotes.

33
metagpt/config.py
View file

@ -8,12 +8,12 @@ Provide configuration, singleton
"""
import os
from copy import deepcopy
from pathlib import Path
from typing import Any
import openai
import yaml
from metagpt.const import OPTIONS, PROJECT_ROOT
from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT, OPTIONS
from metagpt.logs import logger
from metagpt.tools import SearchEngineType, WebBrowserEngineType
from metagpt.utils.singleton import Singleton
@ -40,8 +40,9 @@ class Config(metaclass=Singleton):
"""
_instance = None
key_yaml_file = PROJECT_ROOT / "config/key.yaml"
default_yaml_file = PROJECT_ROOT / "config/config.yaml"
home_yaml_file = Path.home() / ".metagpt/config.yaml"
key_yaml_file = METAGPT_ROOT / "config/key.yaml"
default_yaml_file = METAGPT_ROOT / "config/config.yaml"
def __init__(self, yaml_file=default_yaml_file):
self._init_with_config_files_and_env(yaml_file)
@ -49,18 +50,19 @@ class Config(metaclass=Singleton):
self._update()
def _update(self):
# logger.info("Config loading done.")
self.global_proxy = self._get("GLOBAL_PROXY")
self.openai_api_key = self._get("OPENAI_API_KEY")
self.anthropic_api_key = self._get("Anthropic_API_KEY")
if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and (
not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key
self.zhipuai_api_key = self._get("ZHIPUAI_API_KEY")
if (
(not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key)
and (not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key)
and (not self.zhipuai_api_key or "YOUR_API_KEY" == self.zhipuai_api_key)
):
raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first")
raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY or ZHIPUAI_API_KEY first")
self.openai_api_base = self._get("OPENAI_API_BASE")
openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
if openai_proxy:
openai.proxy = openai_proxy
openai.api_base = self.openai_api_base
self.openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy
self.openai_api_type = self._get("OPENAI_API_TYPE")
self.openai_api_version = self._get("OPENAI_API_VERSION")
self.openai_api_rpm = self._get("RPM", 3)
@ -90,6 +92,7 @@ class Config(metaclass=Singleton):
logger.warning("LONG_TERM_MEMORY is True")
self.max_budget = self._get("MAX_BUDGET", 10.0)
self.total_cost = 0.0
self.code_review_k_times = 2
self.puppeteer_config = self._get("PUPPETEER_CONFIG", "")
self.mmdc = self._get("MMDC", "mmdc")
@ -100,12 +103,18 @@ class Config(metaclass=Singleton):
self.pyppeteer_executable_path = self._get("PYPPETEER_EXECUTABLE_PATH", "")
self.prompt_format = self._get("PROMPT_FORMAT", "markdown")
self.workspace_path = Path(self._get("WORKSPACE_PATH", DEFAULT_WORKSPACE_ROOT))
self._ensure_workspace_exists()
def _ensure_workspace_exists(self):
self.workspace_path.mkdir(parents=True, exist_ok=True)
logger.info(f"WORKSPACE_PATH set to {self.workspace_path}")
def _init_with_config_files_and_env(self, yaml_file):
"""Load from config/key.yaml, config/config.yaml, and env in decreasing order of priority"""
configs = dict(os.environ)
for _yaml_file in [yaml_file, self.key_yaml_file]:
for _yaml_file in [yaml_file, self.key_yaml_file, self.home_yaml_file]:
if not _yaml_file.exists():
continue

99
metagpt/const.py
View file

@ -9,45 +9,90 @@
@Modified By: mashenquan, 2023-11-27. Defines file repository paths according to Section 2.2.3.4 of RFC 135.
"""
import contextvars
import os
from pathlib import Path
from loguru import logger
def get_project_root():
"""Search upwards to find the project root directory."""
current_path = Path.cwd()
while True:
if (
(current_path / ".git").exists()
or (current_path / ".project_root").exists()
or (current_path / ".gitignore").exists()
):
return current_path
parent_path = current_path.parent
if parent_path == current_path:
raise Exception("Project root not found.")
current_path = parent_path
import metagpt
OPTIONS = contextvars.ContextVar("OPTIONS")
PROJECT_ROOT = get_project_root()
DATA_PATH = PROJECT_ROOT / "data"
WORKSPACE_ROOT = PROJECT_ROOT / "workspace"
PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts"
UT_PATH = PROJECT_ROOT / "data/ut"
SWAGGER_PATH = UT_PATH / "files/api/"
UT_PY_PATH = UT_PATH / "files/ut/"
API_QUESTIONS_PATH = UT_PATH / "files/question/"
YAPI_URL = "http://yapi.deepwisdomai.com/"
TMP = PROJECT_ROOT / "tmp"
# <<<<<<< HEAD
# def get_project_root():
# """Search upwards to find the project root directory."""
# current_path = Path.cwd()
# while True:
# if (
# (current_path / ".git").exists()
# or (current_path / ".project_root").exists()
# or (current_path / ".gitignore").exists()
# ):
# return current_path
# parent_path = current_path.parent
# if parent_path == current_path:
# raise Exception("Project root not found.")
# current_path = parent_path
#
#
# PROJECT_ROOT = get_project_root()
# DATA_PATH = PROJECT_ROOT / "data"
# WORKSPACE_ROOT = PROJECT_ROOT / "workspace"
# PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts"
# UT_PATH = PROJECT_ROOT / "data/ut"
# SWAGGER_PATH = UT_PATH / "files/api/"
# UT_PY_PATH = UT_PATH / "files/ut/"
# API_QUESTIONS_PATH = UT_PATH / "files/question/"
# YAPI_URL = "http://yapi.deepwisdomai.com/"
# TMP = PROJECT_ROOT / "tmp"
# =======
def get_metagpt_package_root():
"""Get the root directory of the installed package."""
package_root = Path(metagpt.__file__).parent.parent
logger.info(f"Package root set to {str(package_root)}")
return package_root
def get_metagpt_root():
"""Get the project root directory."""
# Check if a project root is specified in the environment variable
project_root_env = os.getenv("METAGPT_PROJECT_ROOT")
if project_root_env:
project_root = Path(project_root_env)
logger.info(f"PROJECT_ROOT set from environment variable to {str(project_root)}")
else:
# Fallback to package root if no environment variable is set
project_root = get_metagpt_package_root()
return project_root
# METAGPT PROJECT ROOT AND VARS
METAGPT_ROOT = get_metagpt_root()
DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace"
DATA_PATH = METAGPT_ROOT / "data"
RESEARCH_PATH = DATA_PATH / "research"
TUTORIAL_PATH = DATA_PATH / "tutorial_docx"
INVOICE_OCR_TABLE_PATH = DATA_PATH / "invoice_table"
UT_PATH = DATA_PATH / "ut"
SWAGGER_PATH = UT_PATH / "files/api/"
UT_PY_PATH = UT_PATH / "files/ut/"
API_QUESTIONS_PATH = UT_PATH / "files/question/"
SKILL_DIRECTORY = PROJECT_ROOT / "metagpt/skills"
TMP = METAGPT_ROOT / "tmp"
SOURCE_ROOT = METAGPT_ROOT / "metagpt"
PROMPT_PATH = SOURCE_ROOT / "prompts"
SKILL_DIRECTORY = SOURCE_ROOT / "skills"
# REAL CONSTS
MEM_TTL = 24 * 30 * 3600
MESSAGE_ROUTE_FROM = "sent_from"
MESSAGE_ROUTE_TO = "send_to"
MESSAGE_ROUTE_CAUSE_BY = "cause_by"
@ -70,3 +115,5 @@ PRD_PDF_FILE_REPO = "resources/prd"
TASK_PDF_FILE_REPO = "resources/api_spec_and_tasks"
TEST_CODES_FILE_REPO = "tests"
TEST_OUTPUTS_FILE_REPO = "test_outputs"
YAPI_URL = "http://yapi.deepwisdomai.com/"

255
metagpt/document.py Normal file
View file

@ -0,0 +1,255 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/6/8 14:03
@Author : alexanderwu
@File : document.py
"""
from enum import Enum
from pathlib import Path
from typing import Optional, Union
import pandas as pd
from langchain.document_loaders import (
TextLoader,
UnstructuredPDFLoader,
UnstructuredWordDocumentLoader,
)
from langchain.text_splitter import CharacterTextSplitter
from pydantic import BaseModel, Field
from tqdm import tqdm
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.repo_parser import RepoParser
def validate_cols(content_col: str, df: pd.DataFrame):
if content_col not in df.columns:
raise ValueError("Content column not found in DataFrame.")
def read_data(data_path: Path):
suffix = data_path.suffix
if ".xlsx" == suffix:
data = pd.read_excel(data_path)
elif ".csv" == suffix:
data = pd.read_csv(data_path)
elif ".json" == suffix:
data = pd.read_json(data_path)
elif suffix in (".docx", ".doc"):
data = UnstructuredWordDocumentLoader(str(data_path), mode="elements").load()
elif ".txt" == suffix:
data = TextLoader(str(data_path)).load()
text_splitter = CharacterTextSplitter(separator="\n", chunk_size=256, chunk_overlap=0)
texts = text_splitter.split_documents(data)
data = texts
elif ".pdf" == suffix:
data = UnstructuredPDFLoader(str(data_path), mode="elements").load()
else:
raise NotImplementedError("File format not supported.")
return data
class DocumentStatus(Enum):
"""Indicates document status, a mechanism similar to RFC/PEP"""
DRAFT = "draft"
UNDERREVIEW = "underreview"
APPROVED = "approved"
DONE = "done"
class Document(BaseModel):
"""
Document: Handles operations related to document files.
"""
path: Path = Field(default=None)
name: str = Field(default="")
content: str = Field(default="")
# metadata? in content perhaps.
author: str = Field(default="")
status: DocumentStatus = Field(default=DocumentStatus.DRAFT)
reviews: list = Field(default_factory=list)
@classmethod
def from_path(cls, path: Path):
"""
Create a Document instance from a file path.
"""
if not path.exists():
raise FileNotFoundError(f"File {path} not found.")
content = path.read_text()
return cls(content=content, path=path)
@classmethod
def from_text(cls, text: str, path: Optional[Path] = None):
"""
Create a Document from a text string.
"""
return cls(content=text, path=path)
def to_path(self, path: Optional[Path] = None):
"""
Save content to the specified file path.
"""
if path is not None:
self.path = path
if self.path is None:
raise ValueError("File path is not set.")
self.path.parent.mkdir(parents=True, exist_ok=True)
self.path.write_text(self.content, encoding="utf-8")
def persist(self):
"""
Persist document to disk.
"""
return self.to_path()
class IndexableDocument(Document):
"""
Advanced document handling: For vector databases or search engines.
"""
data: Union[pd.DataFrame, list]
content_col: Optional[str] = Field(default="")
meta_col: Optional[str] = Field(default="")
class Config:
arbitrary_types_allowed = True
@classmethod
def from_path(cls, data_path: Path, content_col="content", meta_col="metadata"):
if not data_path.exists():
raise FileNotFoundError(f"File {data_path} not found.")
data = read_data(data_path)
content = data_path.read_text()
if isinstance(data, pd.DataFrame):
validate_cols(content_col, data)
return cls(data=data, content=content, content_col=content_col, meta_col=meta_col)
def _get_docs_and_metadatas_by_df(self) -> (list, list):
df = self.data
docs = []
metadatas = []
for i in tqdm(range(len(df))):
docs.append(df[self.content_col].iloc[i])
if self.meta_col:
metadatas.append({self.meta_col: df[self.meta_col].iloc[i]})
else:
metadatas.append({})
return docs, metadatas
def _get_docs_and_metadatas_by_langchain(self) -> (list, list):
data = self.data
docs = [i.page_content for i in data]
metadatas = [i.metadata for i in data]
return docs, metadatas
def get_docs_and_metadatas(self) -> (list, list):
if isinstance(self.data, pd.DataFrame):
return self._get_docs_and_metadatas_by_df()
elif isinstance(self.data, list):
return self._get_docs_and_metadatas_by_langchain()
else:
raise NotImplementedError("Data type not supported for metadata extraction.")
class RepoMetadata(BaseModel):
name: str = Field(default="")
n_docs: int = Field(default=0)
n_chars: int = Field(default=0)
symbols: list = Field(default_factory=list)
class Repo(BaseModel):
# Name of this repo.
name: str = Field(default="")
# metadata: RepoMetadata = Field(default=RepoMetadata)
docs: dict[Path, Document] = Field(default_factory=dict)
codes: dict[Path, Document] = Field(default_factory=dict)
assets: dict[Path, Document] = Field(default_factory=dict)
path: Path = Field(default=None)
def _path(self, filename):
return self.path / filename
@classmethod
def from_path(cls, path: Path):
"""Load documents, code, and assets from a repository path."""
path.mkdir(parents=True, exist_ok=True)
repo = Repo(path=path, name=path.name)
for file_path in path.rglob("*"):
# FIXME: These judgments are difficult to support multiple programming languages and need to be more general
if file_path.is_file() and file_path.suffix in [".json", ".txt", ".md", ".py", ".js", ".css", ".html"]:
repo._set(file_path.read_text(), file_path)
return repo
def to_path(self):
"""Persist all documents, code, and assets to the given repository path."""
for doc in self.docs.values():
doc.to_path()
for code in self.codes.values():
code.to_path()
for asset in self.assets.values():
asset.to_path()
def _set(self, content: str, path: Path):
"""Add a document to the appropriate category based on its file extension."""
suffix = path.suffix
doc = Document(content=content, path=path, name=str(path.relative_to(self.path)))
# FIXME: These judgments are difficult to support multiple programming languages and need to be more general
if suffix.lower() == ".md":
self.docs[path] = doc
elif suffix.lower() in [".py", ".js", ".css", ".html"]:
self.codes[path] = doc
else:
self.assets[path] = doc
return doc
def set(self, content: str, filename: str):
"""Set a document and persist it to disk."""
path = self._path(filename)
doc = self._set(content, path)
doc.to_path()
def get(self, filename: str) -> Optional[Document]:
"""Get a document by its filename."""
path = self._path(filename)
return self.docs.get(path) or self.codes.get(path) or self.assets.get(path)
def get_text_documents(self) -> list[Document]:
return list(self.docs.values()) + list(self.codes.values())
def eda(self) -> RepoMetadata:
n_docs = sum(len(i) for i in [self.docs, self.codes, self.assets])
n_chars = sum(sum(len(j.content) for j in i.values()) for i in [self.docs, self.codes, self.assets])
symbols = RepoParser(base_directory=self.path).generate_symbols()
return RepoMetadata(name=self.name, n_docs=n_docs, n_chars=n_chars, symbols=symbols)
def set_existing_repo(path=CONFIG.workspace_path / "t1"):
repo1 = Repo.from_path(path)
repo1.set("wtf content", "doc/wtf_file.md")
repo1.set("wtf code", "code/wtf_file.py")
logger.info(repo1) # check doc
def load_existing_repo(path=CONFIG.workspace_path / "web_tetris"):
repo = Repo.from_path(path)
logger.info(repo)
logger.info(repo.eda())
def main():
load_existing_repo()
if __name__ == "__main__":
main()

10
metagpt/document_store/base_store.py
View file

@ -28,20 +28,20 @@ class BaseStore(ABC):
class LocalStore(BaseStore, ABC):
def __init__(self, raw_data: Path, cache_dir: Path = None):
if not raw_data:
def __init__(self, raw_data_path: Path, cache_dir: Path = None):
if not raw_data_path:
raise FileNotFoundError
self.config = Config()
self.raw_data = raw_data
self.raw_data_path = raw_data_path
if not cache_dir:
cache_dir = raw_data.parent
cache_dir = raw_data_path.parent
self.cache_dir = cache_dir
self.store = self._load()
if not self.store:
self.store = self.write()
def _get_index_and_store_fname(self):
fname = self.raw_data.name.split(".")[0]
fname = self.raw_data_path.name.split(".")[0]
index_file = self.cache_dir / f"{fname}.index"
store_file = self.cache_dir / f"{fname}.pkl"
return index_file, store_file

10
metagpt/document_store/faiss_store.py
View file

@ -14,16 +14,16 @@ from langchain.embeddings import OpenAIEmbeddings
from langchain.vectorstores import FAISS
from metagpt.const import DATA_PATH
from metagpt.document import IndexableDocument
from metagpt.document_store.base_store import LocalStore
from metagpt.document_store.document import Document
from metagpt.logs import logger
class FaissStore(LocalStore):
def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"):
def __init__(self, raw_data_path: Path, cache_dir=None, meta_col="source", content_col="output"):
self.meta_col = meta_col
self.content_col = content_col
super().__init__(raw_data, cache_dir)
super().__init__(raw_data_path, cache_dir)
def _load(self) -> Optional["FaissStore"]:
index_file, store_file = self._get_index_and_store_fname()
@ -60,9 +60,9 @@ class FaissStore(LocalStore):
def write(self):
"""Initialize the index and library based on the Document (JSON / XLSX, etc.) file provided by the user."""
if not self.raw_data.exists():
if not self.raw_data_path.exists():
raise FileNotFoundError
doc = Document(self.raw_data, self.content_col, self.meta_col)
doc = IndexableDocument.from_path(self.raw_data_path, self.content_col, self.meta_col)
docs, metadatas = doc.get_docs_and_metadatas()
self.store = self._write(docs, metadatas)

44
metagpt/environment.py
View file

@ -23,6 +23,7 @@ from metagpt.utils.common import is_subscribed
class Environment(BaseModel):
# <<<<<<< HEAD
"""环境，承载一批角色，角色可以向环境发布消息，可以被其他角色观察到
Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles
@ -31,6 +32,17 @@ class Environment(BaseModel):
roles: dict[str, Role] = Field(default_factory=dict)
members: dict[Role, Set] = Field(default_factory=dict)
history: str = Field(default="") # For debug
# =======
# """
# Environment, hosting a batch of roles, roles can publish messages to the environment, and can be observed by other roles
# """
#
# roles: dict[str, Role] = Field(default_factory=dict)
# memory: Memory = Field(default_factory=Memory) # 已经私有化
# history: str = Field(default='')
# repo: Repo = Field(default_factory=Repo) # 在CONFIG里
# kv: dict = Field(default_factory=dict) # 在CONFIG里
# >>>>>>> feature/geekan_cli_etc
class Config:
arbitrary_types_allowed = True
@ -71,6 +83,38 @@ class Environment(BaseModel):
return True
# # Replaced by FileRepository.set_file
# def set_doc(self, content: str, filename: str):
# """向当前环境发布文档（包括代码）"""
# return self.repo.set(content, filename)
#
# # Replaced by FileRepository.get_file
# def get_doc(self, filename: str):
# return self.repo.get(filename)
#
# # Replaced by CONFIG.xx
# def set(self, k: str, v: str):
# self.kv[k] = v
#
# # Replaced by CONFIG.xx
# def get(self, k: str):
# return self.kv.get(k, None)
# Replaced By 增量变更流程
# def load_existing_repo(self, path: Path, inc: bool):
# self.repo = Repo.from_path(path)
# logger.info(self.repo.eda())
#
# # Incremental mode: publish all docs to messages. Then roles can read the docs.
# if inc:
# docs = self.repo.get_text_documents()
# for doc in docs:
# msg = Message(content=doc.content)
# self.publish_message(msg)
# logger.info(f"Message from existing doc {doc.path}: {msg}")
# logger.info(f"Load {len(docs)} docs from existing repo.")
# raise NotImplementedError
async def run(self, k=1):
"""处理一次所有信息的运行
Process all Role runs at once

28
metagpt/llm.py
View file

@ -6,15 +6,25 @@
@File : llm.py
"""
from metagpt.config import CONFIG
from metagpt.provider.anthropic_api import Claude2 as Claude
from metagpt.provider.openai_api import OpenAIGPTAPI as LLM
DEFAULT_LLM = LLM()
CLAUDE_LLM = Claude()
from metagpt.provider.openai_api import OpenAIGPTAPI
from metagpt.provider.spark_api import SparkAPI
from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI
async def ai_func(prompt):
"""使用LLM进行QA
QA with LLMs
"""
return await DEFAULT_LLM.aask(prompt)
def LLM() -> "BaseGPTAPI":
"""initialize different LLM instance according to the key field existence"""
# TODO a little trick, can use registry to initialize LLM instance further
if CONFIG.openai_api_key:
llm = OpenAIGPTAPI()
elif CONFIG.claude_api_key:
llm = Claude()
elif CONFIG.spark_api_key:
llm = SparkAPI()
elif CONFIG.zhipuai_api_key:
llm = ZhiPuAIGPTAPI()
else:
raise RuntimeError("You should config a LLM configuration first")
return llm

8
metagpt/logs.py
View file

@ -10,16 +10,14 @@ import sys
from loguru import logger as _logger
from metagpt.const import PROJECT_ROOT
from metagpt.const import METAGPT_ROOT
def define_log_level(print_level="INFO", logfile_level="DEBUG"):
"""调整日志级别到level之上
Adjust the log level to above level
"""
"""Adjust the log level to above level"""
_logger.remove()
_logger.add(sys.stderr, level=print_level)
_logger.add(PROJECT_ROOT / "logs/log.txt", level=logfile_level)
_logger.add(METAGPT_ROOT / "logs/log.txt", level=logfile_level)
return _logger

2
metagpt/manager.py
View file

@ -14,7 +14,7 @@ class Manager:
def __init__(self, llm: LLM = LLM()):
self.llm = llm # Large Language Model
self.role_directions = {
"BOSS": "Product Manager",
"User": "Product Manager",
"Product Manager": "Architect",
"Architect": "Engineer",
"Engineer": "QA Engineer",

5
metagpt/memory/__init__.py
View file

@ -7,10 +7,11 @@
"""
from metagpt.memory.memory import Memory
from metagpt.memory.longterm_memory import LongTermMemory
# from metagpt.memory.longterm_memory import LongTermMemory
__all__ = [
"Memory",
"LongTermMemory",
# "LongTermMemory",
]

2
metagpt/memory/longterm_memory.py
View file

@ -30,7 +30,7 @@ class LongTermMemory(Memory):
logger.warning(f"It may the first time to run Agent {role_id}, the long-term memory is empty")
else:
logger.warning(
f"Agent {role_id} has existed memory storage with {len(messages)} messages " f"and has recovered them."
f"Agent {role_id} has existing memory storage with {len(messages)} messages " f"and has recovered them."
)
self.msg_from_recover = True
self.add_batch(messages)

1
metagpt/provider/base_chatbot.py
View file

@ -14,6 +14,7 @@ class BaseChatbot(ABC):
"""Abstract GPT class"""
mode: str = "API"
use_system_prompt: bool = True
@abstractmethod
def ask(self, msg: str) -> str:

53
metagpt/provider/base_gpt_api.py
View file

@ -5,6 +5,7 @@
@Author : alexanderwu
@File : base_gpt_api.py
"""
import json
from abc import abstractmethod
from typing import Optional
@ -33,15 +34,21 @@ class BaseGPTAPI(BaseChatbot):
return self._system_msg(self.system_prompt)
def ask(self, msg: str) -> str:
message = [self._default_system_msg(), self._user_msg(msg)]
message = [self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)]
rsp = self.completion(message)
return self.get_choice_text(rsp)
async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str:
if system_msgs:
message = self._system_msgs(system_msgs) + [self._user_msg(msg)]
message = (
self._system_msgs(system_msgs) + [self._user_msg(msg)]
if self.use_system_prompt
else [self._user_msg(msg)]
)
else:
message = [self._default_system_msg(), self._user_msg(msg)]
message = (
[self._default_system_msg(), self._user_msg(msg)] if self.use_system_prompt else [self._user_msg(msg)]
)
rsp = await self.acompletion_text(message, stream=True)
logger.debug(message)
# logger.debug(rsp)
@ -109,6 +116,46 @@ class BaseGPTAPI(BaseChatbot):
"""Required to provide the first text of choice"""
return rsp.get("choices")[0]["message"]["content"]
def get_choice_function(self, rsp: dict) -> dict:
"""Required to provide the first function of choice
:param dict rsp: OpenAI chat.comletion respond JSON, Note "message" must include "tool_calls",
and "tool_calls" must include "function", for example:
{...
"choices": [
{
"index": 0,
"message": {
"role": "assistant",
"content": null,
"tool_calls": [
{
"id": "call_Y5r6Ddr2Qc2ZrqgfwzPX5l72",
"type": "function",
"function": {
"name": "execute",
"arguments": "{\n \"language\": \"python\",\n \"code\": \"print('Hello, World!')\"\n}"
}
}
]
},
"finish_reason": "stop"
}
],
...}
:return dict: return first function of choice, for exmaple,
{'name': 'execute', 'arguments': '{\n "language": "python",\n "code": "print(\'Hello, World!\')"\n}'}
"""
return rsp.get("choices")[0]["message"]["tool_calls"][0]["function"].to_dict()
def get_choice_function_arguments(self, rsp: dict) -> dict:
"""Required to provide the first function arguments of choice.
:param dict rsp: same as in self.get_choice_function(rsp)
:return dict: return the first function arguments of choice, for example,
{'language': 'python', 'code': "print('Hello, World!')"}
"""
return json.loads(self.get_choice_function(rsp)["arguments"])
def messages_to_prompt(self, messages: list[dict]):
"""[{"role": "user", "content": msg}] to user: <msg> etc."""
return "\n".join([f"{i['role']}: {i['content']}" for i in messages])

30
metagpt/provider/constant.py Normal file
View file

@ -0,0 +1,30 @@
# function in tools, https://platform.openai.com/docs/api-reference/chat/create#chat-create-tools
# Reference: https://github.com/KillianLucas/open-interpreter/blob/v0.1.14/interpreter/llm/setup_openai_coding_llm.py
GENERAL_FUNCTION_SCHEMA = {
"name": "execute",
"description": "Executes code on the user's machine, **in the users local environment**, and returns the output",
"parameters": {
"type": "object",
"properties": {
"language": {
"type": "string",
"description": "The programming language (required parameter to the `execute` function)",
"enum": [
"python",
"R",
"shell",
"applescript",
"javascript",
"html",
"powershell",
],
},
"code": {"type": "string", "description": "The code to execute (required)"},
},
"required": ["language", "code"],
},
}
# tool_choice value for general_function_schema
# https://platform.openai.com/docs/api-reference/chat/create#chat-create-tool_choice
GENERAL_TOOL_CHOICE = {"type": "function", "function": {"name": "execute"}}

58
metagpt/provider/general_api_requestor.py Normal file
View file

@ -0,0 +1,58 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : General Async API for http-based LLM model
import asyncio
from typing import AsyncGenerator, Tuple, Union
import aiohttp
from openai.api_requestor import APIRequestor
from metagpt.logs import logger
class GeneralAPIRequestor(APIRequestor):
"""
usage
# full_url = "{api_base}{url}"
requester = GeneralAPIRequestor(api_base=api_base)
result, _, api_key = await requester.arequest(
method=method,
url=url,
headers=headers,
stream=stream,
params=kwargs,
request_timeout=120
)
"""
def _interpret_response_line(self, rbody: str, rcode: int, rheaders, stream: bool) -> str:
# just do nothing to meet the APIRequestor process and return the raw data
# due to the openai sdk will convert the data into OpenAIResponse which we don't need in general cases.
return rbody
async def _interpret_async_response(
self, result: aiohttp.ClientResponse, stream: bool
) -> Tuple[Union[str, AsyncGenerator[str, None]], bool]:
if stream and "text/event-stream" in result.headers.get("Content-Type", ""):
return (
self._interpret_response_line(line, result.status, result.headers, stream=True)
async for line in result.content
), True
else:
try:
await result.read()
except (aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e:
raise TimeoutError("Request timed out") from e
except aiohttp.ClientError as exp:
logger.warning(f"response: {result.content}, exp: {exp}")
return (
self._interpret_response_line(
await result.read(), # let the caller to decode the msg
result.status,
result.headers,
stream=False,
),
False,
)

37
metagpt/provider/human_provider.py Normal file
View file

@ -0,0 +1,37 @@
"""
Filename: MetaGPT/metagpt/provider/human_provider.py
Created Date: Wednesday, November 8th 2023, 11:55:46 pm
Author: garylin2099
"""
from typing import Optional
from metagpt.logs import logger
from metagpt.provider.base_gpt_api import BaseGPTAPI
class HumanProvider(BaseGPTAPI):
"""Humans provide themselves as a 'model', which actually takes in human input as its response.
This enables replacing LLM anywhere in the framework with a human, thus introducing human interaction
"""
def ask(self, msg: str) -> str:
logger.info("It's your turn, please type in your response. You may also refer to the context below")
rsp = input(msg)
if rsp in ["exit", "quit"]:
exit()
return rsp
async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str:
return self.ask(msg)
def completion(self, messages: list[dict]):
"""dummy implementation of abstract method in base"""
return []
async def acompletion(self, messages: list[dict]):
"""dummy implementation of abstract method in base"""
return []
async def acompletion_text(self, messages: list[dict], stream=False) -> str:
"""dummy implementation of abstract method in base"""
return []

84
metagpt/provider/openai_api.py
View file

@ -21,6 +21,8 @@ from tenacity import (
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.provider.base_gpt_api import BaseGPTAPI
from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE
from metagpt.schema import Message
from metagpt.utils.singleton import Singleton
from metagpt.utils.token_counter import (
TOKEN_COSTS,
@ -155,6 +157,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
if config.openai_api_type:
openai.api_type = config.openai_api_type
openai.api_version = config.openai_api_version
if config.openai_proxy:
openai.proxy = config.openai_proxy
self.rpm = int(config.get("RPM", 10))
async def _achat_completion_stream(self, messages: list[dict]) -> str:
@ -179,7 +183,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
self._update_costs(usage)
return full_reply_content
def _cons_kwargs(self, messages: list[dict]) -> dict:
def _cons_kwargs(self, messages: list[dict], **configs) -> dict:
kwargs = {
"messages": messages,
"max_tokens": self.get_max_tokens(messages),
@ -188,6 +192,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
"temperature": 0.3,
"timeout": 3,
}
if configs:
kwargs.update(configs)
if CONFIG.openai_api_type == "azure":
if CONFIG.deployment_name and CONFIG.deployment_id:
raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model")
@ -237,6 +244,81 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
rsp = await self._achat_completion(messages)
return self.get_choice_text(rsp)
def _func_configs(self, messages: list[dict], **kwargs) -> dict:
"""
Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
"""
if "tools" not in kwargs:
configs = {
"tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}],
"tool_choice": GENERAL_TOOL_CHOICE,
}
kwargs.update(configs)
return self._cons_kwargs(messages, **kwargs)
def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict:
rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs))
self._update_costs(rsp.get("usage"))
return rsp
async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict:
rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs))
self._update_costs(rsp.get("usage"))
return rsp
def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]:
"""convert messages to list[dict]."""
if isinstance(messages, list):
messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages]
return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages]
if isinstance(messages, Message):
messages = [messages.to_dict()]
elif isinstance(messages, str):
messages = [{"role": "user", "content": messages}]
else:
raise ValueError(
f"Only support messages type are: str, Message, list[dict], but got {type(messages).__name__}!"
)
return messages
def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict:
"""Use function of tools to ask a code.
Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
Examples:
>>> llm = OpenAIGPTAPI()
>>> llm.ask_code("Write a python hello world code.")
{'language': 'python', 'code': "print('Hello, World!')"}
>>> msg = [{'role': 'user', 'content': "Write a python hello world code."}]
>>> llm.ask_code(msg)
{'language': 'python', 'code': "print('Hello, World!')"}
"""
messages = self._process_message(messages)
rsp = self._chat_completion_function(messages, **kwargs)
return self.get_choice_function_arguments(rsp)
async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict:
"""Use function of tools to ask a code.
Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
Examples:
>>> llm = OpenAIGPTAPI()
>>> rsp = await llm.ask_code("Write a python hello world code.")
>>> rsp
{'language': 'python', 'code': "print('Hello, World!')"}
>>> msg = [{'role': 'user', 'content': "Write a python hello world code."}]
>>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"}
"""
messages = self._process_message(messages)
rsp = await self._achat_completion_function(messages, **kwargs)
return self.get_choice_function_arguments(rsp)
def _calc_usage(self, messages: list[dict], rsp: str) -> dict:
usage = {}
if CONFIG.calc_usage:

3
metagpt/provider/zhipuai/__init__.py Normal file
View file

@ -0,0 +1,3 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc :

75
metagpt/provider/zhipuai/async_sse_client.py Normal file
View file

@ -0,0 +1,75 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : async_sse_client to make keep the use of Event to access response
# refs to `https://github.com/zhipuai/zhipuai-sdk-python/blob/main/zhipuai/utils/sse_client.py`
from zhipuai.utils.sse_client import _FIELD_SEPARATOR, Event, SSEClient
class AsyncSSEClient(SSEClient):
async def _aread(self):
data = b""
async for chunk in self._event_source:
for line in chunk.splitlines(True):
data += line
if data.endswith((b"\r\r", b"\n\n", b"\r\n\r\n")):
yield data
data = b""
if data:
yield data
async def async_events(self):
async for chunk in self._aread():
event = Event()
# Split before decoding so splitlines() only uses \r and \n
for line in chunk.splitlines():
# Decode the line.
line = line.decode(self._char_enc)
# Lines starting with a separator are comments and are to be
# ignored.
if not line.strip() or line.startswith(_FIELD_SEPARATOR):
continue
data = line.split(_FIELD_SEPARATOR, 1)
field = data[0]
# Ignore unknown fields.
if field not in event.__dict__:
self._logger.debug("Saw invalid field %s while parsing " "Server Side Event", field)
continue
if len(data) > 1:
# From the spec:
# "If value starts with a single U+0020 SPACE character,
# remove it from value."
if data[1].startswith(" "):
value = data[1][1:]
else:
value = data[1]
else:
# If no value is present after the separator,
# assume an empty value.
value = ""
# The data field may come over multiple lines and their values
# are concatenated with each other.
if field == "data":
event.__dict__[field] += value + "\n"
else:
event.__dict__[field] = value
# Events with no data are not dispatched.
if not event.data:
continue
# If the data field ends with a newline, remove it.
if event.data.endswith("\n"):
event.data = event.data[0:-1]
# Empty event names default to 'message'
event.event = event.event or "message"
# Dispatch the event
self._logger.debug("Dispatching %s...", event)
yield event

72
metagpt/provider/zhipuai/zhipu_model_api.py Normal file
View file

@ -0,0 +1,72 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : zhipu model api to support sync & async for invoke & sse_invoke
import zhipuai
from zhipuai.model_api.api import InvokeType, ModelAPI
from zhipuai.utils.http_client import headers as zhipuai_default_headers
from metagpt.provider.general_api_requestor import GeneralAPIRequestor
from metagpt.provider.zhipuai.async_sse_client import AsyncSSEClient
class ZhiPuModelAPI(ModelAPI):
@classmethod
def get_header(cls) -> dict:
token = cls._generate_token()
zhipuai_default_headers.update({"Authorization": token})
return zhipuai_default_headers
@classmethod
def get_sse_header(cls) -> dict:
token = cls._generate_token()
headers = {"Authorization": token}
return headers
@classmethod
def split_zhipu_api_url(cls, invoke_type: InvokeType, kwargs):
# use this method to prevent zhipu api upgrading to different version.
# and follow the GeneralAPIRequestor implemented based on openai sdk
zhipu_api_url = cls._build_api_url(kwargs, invoke_type)
"""
example:
zhipu_api_url: https://open.bigmodel.cn/api/paas/v3/model-api/{model}/{invoke_method}
"""
arr = zhipu_api_url.split("/api/")
# ("https://open.bigmodel.cn/api/" , "/paas/v3/model-api/chatglm_turbo/invoke")
return f"{arr[0]}/api", f"/{arr[1]}"
@classmethod
async def arequest(cls, invoke_type: InvokeType, stream: bool, method: str, headers: dict, kwargs):
# TODO to make the async request to be more generic for models in http mode.
assert method in ["post", "get"]
api_base, url = cls.split_zhipu_api_url(invoke_type, kwargs)
requester = GeneralAPIRequestor(api_base=api_base)
result, _, api_key = await requester.arequest(
method=method,
url=url,
headers=headers,
stream=stream,
params=kwargs,
request_timeout=zhipuai.api_timeout_seconds,
)
return result
@classmethod
async def ainvoke(cls, **kwargs) -> dict:
"""async invoke different from raw method `async_invoke` which get the final result by task_id"""
headers = cls.get_header()
resp = await cls.arequest(
invoke_type=InvokeType.SYNC, stream=False, method="post", headers=headers, kwargs=kwargs
)
return resp
@classmethod
async def asse_invoke(cls, **kwargs) -> AsyncSSEClient:
"""async sse_invoke"""
headers = cls.get_sse_header()
return AsyncSSEClient(
await cls.arequest(invoke_type=InvokeType.SSE, stream=True, method="post", headers=headers, kwargs=kwargs)
)

135
metagpt/provider/zhipuai_api.py Normal file
View file

@ -0,0 +1,135 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : zhipuai LLM from https://open.bigmodel.cn/dev/api#sdk
import json
from enum import Enum
import openai
import zhipuai
from requests import ConnectionError
from tenacity import (
after_log,
retry,
retry_if_exception_type,
stop_after_attempt,
wait_fixed,
)
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.provider.base_gpt_api import BaseGPTAPI
from metagpt.provider.openai_api import CostManager, log_and_reraise
from metagpt.provider.zhipuai.zhipu_model_api import ZhiPuModelAPI
class ZhiPuEvent(Enum):
ADD = "add"
ERROR = "error"
INTERRUPTED = "interrupted"
FINISH = "finish"
class ZhiPuAIGPTAPI(BaseGPTAPI):
"""
Refs to `https://open.bigmodel.cn/dev/api#chatglm_turbo`
From now, there is only one model named `chatglm_turbo`
"""
use_system_prompt: bool = False # zhipuai has no system prompt when use api
def __init__(self):
self.__init_zhipuai(CONFIG)
self.llm = ZhiPuModelAPI
self.model = "chatglm_turbo" # so far only one model, just use it
self._cost_manager = CostManager()
def __init_zhipuai(self, config: CONFIG):
assert config.zhipuai_api_key
zhipuai.api_key = config.zhipuai_api_key
openai.api_key = zhipuai.api_key # due to use openai sdk, set the api_key but it will't be used.
def _const_kwargs(self, messages: list[dict]) -> dict:
kwargs = {"model": self.model, "prompt": messages, "temperature": 0.3}
return kwargs
def _update_costs(self, usage: dict):
"""update each request's token cost"""
if CONFIG.calc_usage:
try:
prompt_tokens = int(usage.get("prompt_tokens", 0))
completion_tokens = int(usage.get("completion_tokens", 0))
self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
except Exception as e:
logger.error("zhipuai updats costs failed!", e)
def get_choice_text(self, resp: dict) -> str:
"""get the first text of choice from llm response"""
assist_msg = resp.get("data", {}).get("choices", [{"role": "error"}])[-1]
assert assist_msg["role"] == "assistant"
return assist_msg.get("content")
def completion(self, messages: list[dict]) -> dict:
resp = self.llm.invoke(**self._const_kwargs(messages))
usage = resp.get("data").get("usage")
self._update_costs(usage)
return resp
async def _achat_completion(self, messages: list[dict]) -> dict:
resp = await self.llm.ainvoke(**self._const_kwargs(messages))
usage = resp.get("data").get("usage")
self._update_costs(usage)
return resp
async def acompletion(self, messages: list[dict]) -> dict:
return await self._achat_completion(messages)
async def _achat_completion_stream(self, messages: list[dict]) -> str:
response = await self.llm.asse_invoke(**self._const_kwargs(messages))
collected_content = []
usage = {}
async for event in response.async_events():
if event.event == ZhiPuEvent.ADD.value:
content = event.data
collected_content.append(content)
print(content, end="")
elif event.event == ZhiPuEvent.ERROR.value or event.event == ZhiPuEvent.INTERRUPTED.value:
content = event.data
logger.error(f"event error: {content}", end="")
collected_content.append([content])
elif event.event == ZhiPuEvent.FINISH.value:
"""
event.meta
{
"task_status":"SUCCESS",
"usage":{
"completion_tokens":351,
"prompt_tokens":595,
"total_tokens":946
},
"task_id":"xx",
"request_id":"xxx"
}
"""
meta = json.loads(event.meta)
usage = meta.get("usage")
else:
print(f"zhipuapi else event: {event.data}", end="")
self._update_costs(usage)
full_content = "".join(collected_content)
return full_content
@retry(
stop=stop_after_attempt(3),
wait=wait_fixed(1),
after=after_log(logger, logger.level("WARNING").name),
retry=retry_if_exception_type(ConnectionError),
retry_error_callback=log_and_reraise,
)
async def acompletion_text(self, messages: list[dict], stream=False) -> str:
"""response in async with stream or non-stream mode"""
if stream:
return await self._achat_completion_stream(messages)
resp = await self._achat_completion(messages)
return self.get_choice_text(resp)

94
metagpt/repo_parser.py Normal file
View file

@ -0,0 +1,94 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/11/17 17:58
@Author : alexanderwu
@File : repo_parser.py
"""
import ast
import json
from pathlib import Path
from pprint import pformat
import pandas as pd
from pydantic import BaseModel, Field
from metagpt.config import CONFIG
from metagpt.logs import logger
class RepoParser(BaseModel):
base_directory: Path = Field(default=None)
def parse_file(self, file_path):
"""Parse a Python file in the repository."""
try:
return ast.parse(file_path.read_text()).body
except:
return []
def extract_class_and_function_info(self, tree, file_path):
"""Extract class, function, and global variable information from the AST."""
file_info = {
"file": str(file_path.relative_to(self.base_directory)),
"classes": [],
"functions": [],
"globals": [],
}
for node in tree:
if isinstance(node, ast.ClassDef):
class_methods = [m.name for m in node.body if is_func(m)]
file_info["classes"].append({"name": node.name, "methods": class_methods})
elif is_func(node):
file_info["functions"].append(node.name)
elif isinstance(node, (ast.Assign, ast.AnnAssign)):
for target in node.targets if isinstance(node, ast.Assign) else [node.target]:
if isinstance(target, ast.Name):
file_info["globals"].append(target.id)
return file_info
def generate_symbols(self):
files_classes = []
directory = self.base_directory
for path in directory.rglob("*.py"):
tree = self.parse_file(path)
file_info = self.extract_class_and_function_info(tree, path)
files_classes.append(file_info)
return files_classes
def generate_json_structure(self, output_path):
"""Generate a JSON file documenting the repository structure."""
files_classes = self.generate_symbols()
output_path.write_text(json.dumps(files_classes, indent=4))
def generate_dataframe_structure(self, output_path):
"""Generate a DataFrame documenting the repository structure and save as CSV."""
files_classes = self.generate_symbols()
df = pd.DataFrame(files_classes)
df.to_csv(output_path, index=False)
def generate_structure(self, output_path=None, mode="json"):
"""Generate the structure of the repository as a specified format."""
output_file = self.base_directory / f"{self.base_directory.name}-structure.{mode}"
output_path = Path(output_path) if output_path else output_file
if mode == "json":
self.generate_json_structure(output_path)
elif mode == "csv":
self.generate_dataframe_structure(output_path)
def is_func(node):
return isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef))
def main():
repo_parser = RepoParser(base_directory=CONFIG.workspace_path / "web_2048")
symbols = repo_parser.generate_symbols()
logger.info(pformat(symbols))
if __name__ == "__main__":
main()

269
metagpt/roles/engineer.py
View file

@ -21,11 +21,18 @@ from pathlib import Path
from typing import Set
from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks
from metagpt.actions.summarize_code import SummarizeCode
from metagpt.config import CONFIG
from metagpt.const import SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.const import MESSAGE_ROUTE_TO_NONE, SYSTEM_DESIGN_FILE_REPO, TASK_FILE_REPO
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import CodingContext, Document, Documents, Message
from metagpt.schema import (
CodeSummarizeContext,
CodingContext,
Document,
Documents,
Message,
)
class Engineer(Role):
@ -39,7 +46,6 @@ class Engineer(Role):
constraints (str): Constraints for the engineer.
n_borg (int): Number of borgs.
use_code_review (bool): Whether to use code review.
todos (list): List of tasks.
"""
def __init__(
@ -55,7 +61,8 @@ class Engineer(Role):
super().__init__(name, profile, goal, constraints)
self.use_code_review = use_code_review
self._watch([WriteTasks])
self.todos = []
self.code_todos = []
self.summarize_todos = []
self.n_borg = n_borg
@staticmethod
@ -63,10 +70,100 @@ class Engineer(Role):
m = json.loads(task_msg.content)
return m.get("Task list")
async def _act_sp_precision(self, review=False) -> Set[str]:
# @classmethod
# def parse_tasks(cls, task_msg: Message) -> list[str]:
# if task_msg.instruct_content:
# return task_msg.instruct_content.dict().get("Task list")
# return CodeParser.parse_file_list(block="Task list", text=task_msg.content)
#
# @classmethod
# def parse_code(cls, code_text: str) -> str:
# return CodeParser.parse_code(block="", text=code_text)
#
# @classmethod
# def parse_workspace(cls, system_design_msg: Message) -> str:
# if system_design_msg.instruct_content:
# return system_design_msg.instruct_content.dict().get("project_name").strip().strip("'").strip('"')
# return CodeParser.parse_str(block="project_name", text=system_design_msg.content)
#
# def get_workspace(self) -> Path:
# msg = self._rc.memory.get_by_action(WriteDesign)[-1]
# if not msg:
# return CONFIG.workspace_path / "src"
# workspace = self.parse_workspace(msg)
# # Codes are written in workspace/{package_name}/{package_name}
# return CONFIG.workspace_path / workspace / workspace
#
# def recreate_workspace(self):
# workspace = self.get_workspace()
# try:
# shutil.rmtree(workspace)
# except FileNotFoundError:
# pass # The folder does not exist, but we don't care
# workspace.mkdir(parents=True, exist_ok=True)
#
# def write_file(self, filename: str, code: str):
# workspace = self.get_workspace()
# filename = filename.replace('"', "").replace("\n", "")
# file = workspace / filename
# file.parent.mkdir(parents=True, exist_ok=True)
# file.write_text(code)
# return file
#
# def recv(self, message: Message) -> None:
# self._rc.memory.add(message)
# if message in self._rc.important_memory:
# self.todos = self.parse_tasks(message)
#
# async def _act_mp(self) -> Message:
# # self.recreate_workspace()
# todo_coros = []
# for todo in self.todos:
# todo_coro = WriteCode().run(
# context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo
# )
# todo_coros.append(todo_coro)
#
# rsps = await gather_ordered_k(todo_coros, self.n_borg)
# for todo, code_rsp in zip(self.todos, rsps):
# _ = self.parse_code(code_rsp)
# logger.info(todo)
# logger.info(code_rsp)
# # self.write_file(todo, code)
# msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo))
# self._rc.memory.add(msg)
# del self.todos[0]
#
# logger.info(f"Done {self.get_workspace()} generating.")
# msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo))
# return msg
#
# async def _act_sp(self) -> Message:
# code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
# for todo in self.todos:
# code = await WriteCode().run(context=self._rc.history, filename=todo)
# # logger.info(todo)
# # logger.info(code_rsp)
# # code = self.parse_code(code_rsp)
# file_path = self.write_file(todo, code)
# msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo))
# self._rc.memory.add(msg)
#
# code_msg = todo + FILENAME_CODE_SEP + str(file_path)
# code_msg_all.append(code_msg)
#
# logger.info(f"Done {self.get_workspace()} generating.")
# msg = Message(
# content=MSG_SEP.join(code_msg_all), role=self.profile, cause_by=type(self._rc.todo), send_to="QaEngineer"
# )
# return msg
# async def _act_sp_with_cr(self) -> Message:
# code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
async def _act_sp_with_cr(self, review=False) -> Set[str]:
changed_files = set()
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
for todo in self.todos:
for todo in self.code_todos:
"""
# Select essential information from the historical data to reduce the length of the prompt (summarized from human experience):
1. All from Architect
@ -77,11 +174,7 @@ class Engineer(Role):
coding_context = await todo.run()
# Code review
if review:
try:
coding_context = await WriteCodeReview(context=coding_context, llm=self._llm).run()
except Exception as e:
logger.error("code review failed!", e)
pass
coding_context = await WriteCodeReview(context=coding_context, llm=self._llm).run()
await src_file_repo.save(
coding_context.filename,
dependencies={coding_context.design_doc.root_relative_path, coding_context.task_doc.root_relative_path},
@ -90,6 +183,24 @@ class Engineer(Role):
msg = Message(
content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode
)
# =======
# context = []
# msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode])
# for m in msg:
# context.append(m.content)
# context_str = "\n----------\n".join(context)
# # Write code
# code = await WriteCode().run(context=context_str, filename=todo)
# # Code review
# if self.use_code_review:
# # try:
# rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo)
# code = rewrite_code
# # except Exception as e:
# # logger.error("code review failed!", e)
# file_path = self.write_file(todo, code)
# msg = Message(content=code, role=self.profile, cause_by=WriteCode)
# >>>>>>> feature/geekan_cli_etc
self._rc.memory.add(msg)
changed_files.add(coding_context.code_doc.filename)
@ -97,26 +208,81 @@ class Engineer(Role):
logger.info("Nothing has changed.")
return changed_files
async def _act(self) -> Message:
async def _act(self) -> Message | None:
"""Determines the mode of action based on whether code review is used."""
changed_files = await self._act_sp_precision(review=self.use_code_review)
# Unit tests only.
if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files:
changed_files.add(CONFIG.REQA_FILENAME)
from metagpt.roles import QaEngineer # Avoid circular references.
msg = Message(
content="\n".join(changed_files),
role=self.profile,
cause_by=WriteCodeReview if self.use_code_review else WriteCode,
send_to=QaEngineer,
)
return msg
if self._rc.todo is None:
return None
if isinstance(self._rc.todo, WriteCode):
changed_files = await self._act_sp_with_cr(review=self.use_code_review)
# Unit tests only.
if CONFIG.REQA_FILENAME and CONFIG.REQA_FILENAME not in changed_files:
changed_files.add(CONFIG.REQA_FILENAME)
return Message(
content="\n".join(changed_files),
role=self.profile,
cause_by=WriteCodeReview if self.use_code_review else WriteCode,
send_to="Edward", # The name of QaEngineer
)
if isinstance(self._rc.todo, SummarizeCode):
summaries = []
for todo in self.summarize_todos:
summary = await todo.run()
summaries.append(summary.json(ensure_ascii=False))
return Message(
content="\n".join(summaries),
role=self.profile,
cause_by=SummarizeCode,
send_to=MESSAGE_ROUTE_TO_NONE,
)
return None
async def _think(self) -> Action | None:
if not CONFIG.src_workspace:
CONFIG.src_workspace = CONFIG.git_repo.workdir / CONFIG.git_repo.workdir.name
if not self.code_todos:
await self._new_code_actions()
elif not self.summarize_todos:
await self._new_summarize_actions()
else:
return None
return self._rc.todo # For agent store
@staticmethod
async def _new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
) -> CodingContext:
old_code_doc = await src_file_repo.get(filename)
if not old_code_doc:
old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="")
dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)}
task_doc = None
design_doc = None
for i in dependencies:
if str(i.parent) == TASK_FILE_REPO:
task_doc = task_file_repo.get(i.filename)
elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO:
design_doc = design_file_repo.get(i.filename)
context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc)
return context
@staticmethod
async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency):
context = await Engineer._new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
)
coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json())
return coding_doc
# =======
# async def _act(self) -> Message:
# """Determines the mode of action based on whether code review is used."""
# logger.info(f"{self._setting}: ready to WriteCode")
# if self.use_code_review:
# return await self._act_sp_with_cr()
# return await self._act_sp()
# >>>>>>> feature/geekan_cli_etc
async def _new_code_actions(self):
# Prepare file repos
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
changed_src_files = src_file_repo.changed_files
@ -146,7 +312,7 @@ class Engineer(Role):
f"{changed_files.docs[task_filename].json()}"
)
changed_files.docs[task_filename] = coding_doc
self.todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()]
self.code_todos = [WriteCode(context=i, llm=self._llm) for i in changed_files.docs.values()]
# Code directly modified by the user.
dependency = await CONFIG.git_repo.get_dependency()
for filename in changed_src_files:
@ -160,34 +326,25 @@ class Engineer(Role):
dependency=dependency,
)
changed_files.docs[filename] = coding_doc
self.todos.append(WriteCode(context=coding_doc, llm=self._llm))
self.code_todos.append(WriteCode(context=coding_doc, llm=self._llm))
if self.todos:
self._rc.todo = self.todos[0]
return self._rc.todo # For agent store
if self.code_todos:
self._rc.todo = self.code_todos[0]
@staticmethod
async def _new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
) -> CodingContext:
old_code_doc = await src_file_repo.get(filename)
if not old_code_doc:
old_code_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content="")
dependencies = {Path(i) for i in await dependency.get(old_code_doc.root_relative_path)}
task_doc = None
design_doc = None
for i in dependencies:
if str(i.parent) == TASK_FILE_REPO:
task_doc = task_file_repo.get(i.filename)
elif str(i.parent) == SYSTEM_DESIGN_FILE_REPO:
design_doc = design_file_repo.get(i.filename)
context = CodingContext(filename=filename, design_doc=design_doc, task_doc=task_doc, code_doc=old_code_doc)
return context
@staticmethod
async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency):
context = await Engineer._new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
)
coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json())
return coding_doc
async def _new_summarize_actions(self):
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
changed_src_files = src_file_repo.changed_files
# Generate a SummarizeCode action for each pair of (system_design_doc, task_doc).
summerizations = {}
for filename in changed_src_files:
depenencies = src_file_repo.get_dependency(filename=filename)
ctx = CodeSummarizeContext.loads(filenames=depenencies)
if ctx not in summerizations:
summerizations[ctx] = set()
srcs = summerizations.get(ctx)
srcs.add(filename)
for ctx, filenames in summerizations.items():
ctx.codes_filenames = filenames
self.summarize_todos.append(SummarizeCode(context=ctx, llm=self._llm))
if self.summarize_todos:
self._rc.todo = self.summarize_todos[0]

25
metagpt/roles/invoice_ocr_assistant.py
View file

@ -42,17 +42,7 @@ class InvoiceOCRAssistant(Role):
self.filename = ""
self.origin_query = ""
self.orc_data = None
async def _think(self) -> None:
"""Determine the next action to be taken by the role."""
if self._rc.todo is None:
self._set_state(0)
return
if self._rc.state + 1 < len(self._states):
self._set_state(self._rc.state + 1)
else:
self._rc.todo = None
self._set_react_mode(react_mode="by_order")
async def _act(self) -> Message:
"""Perform an action as determined by the role.
@ -94,16 +84,3 @@ class InvoiceOCRAssistant(Role):
msg = Message(content=content, instruct_content=resp)
self._rc.memory.add(msg)
return msg
async def _react(self) -> Message:
"""Execute the invoice ocr assistant's think and actions.
Returns:
A message containing the final result of the assistant's actions.
"""
while True:
await self._think()
if self._rc.todo is None:
break
msg = await self._act()
return msg

6
metagpt/roles/product_manager.py
View file

@ -6,7 +6,8 @@
@File : product_manager.py
@Modified By: mashenquan, 2023/11/27. Add `PrepareDocuments` action according to Section 2.2.3.5.1 of RFC 135.
"""
from metagpt.actions import BossRequirement, WritePRD
from metagpt.actions import UserRequirement, WritePRD
from metagpt.actions.prepare_documents import PrepareDocuments
from metagpt.config import CONFIG
from metagpt.roles import Role
@ -40,8 +41,9 @@ class ProductManager(Role):
constraints (str): Constraints or limitations for the product manager.
"""
super().__init__(name, profile, goal, constraints)
self._init_actions([PrepareDocuments, WritePRD])
self._watch([BossRequirement, PrepareDocuments])
self._watch([UserRequirement, PrepareDocuments])
async def _think(self) -> None:
"""Decide what to do"""

28
metagpt/roles/qa_engineer.py
View file

@ -13,6 +13,8 @@
to using file references.
"""
from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest
# from metagpt.const import WORKSPACE_ROOT
from metagpt.config import CONFIG
from metagpt.const import (
MESSAGE_ROUTE_TO_NONE,
@ -42,6 +44,32 @@ class QaEngineer(Role):
self.test_round = 0
self.test_round_allowed = test_round_allowed
# <<<<<<< HEAD
# =======
# @classmethod
# def parse_workspace(cls, system_design_msg: Message) -> str:
# if system_design_msg.instruct_content:
# return system_design_msg.instruct_content.dict().get("project_name")
# return CodeParser.parse_str(block="project_name", text=system_design_msg.content)
#
# def get_workspace(self, return_proj_dir=True) -> Path:
# msg = self._rc.memory.get_by_action(WriteDesign)[-1]
# if not msg:
# return CONFIG.workspace_path / "src"
# workspace = self.parse_workspace(msg)
# # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc.
# if return_proj_dir:
# return CONFIG.workspace_path / workspace
# # development codes directory: workspace/{package_name}/{package_name}
# return CONFIG.workspace_path / workspace / workspace
#
# def write_file(self, filename: str, code: str):
# workspace = self.get_workspace() / "tests"
# file = workspace / filename
# file.parent.mkdir(parents=True, exist_ok=True)
# file.write_text(code)
#
# >>>>>>> feature/geekan_cli_etc
async def _write_test(self, message: Message) -> None:
changed_files = message.content.splitlines()
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)

19
metagpt/roles/researcher.py
View file

@ -36,20 +36,11 @@ class Researcher(Role):
):
super().__init__(name, profile, goal, constraints, **kwargs)
self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)])
self._set_react_mode(react_mode="by_order")
self.language = language
if language not in ("en-us", "zh-cn"):
logger.warning(f"The language `{language}` has not been tested, it may not work.")
async def _think(self) -> None:
if self._rc.todo is None:
self._set_state(0)
return
if self._rc.state + 1 < len(self._states):
self._set_state(self._rc.state + 1)
else:
self._rc.todo = None
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
todo = self._rc.todo
@ -78,12 +69,8 @@ class Researcher(Role):
self._rc.memory.add(ret)
return ret
async def _react(self) -> Message:
while True:
await self._think()
if self._rc.todo is None:
break
msg = await self._act()
async def react(self) -> Message:
msg = await super().react()
report = msg.instruct_content
self.write_report(report.topic, report.content)
return msg

171
metagpt/roles/role.py
View file

@ -20,15 +20,18 @@
"""
from __future__ import annotations
from enum import Enum
from typing import Iterable, Set, Type
from pydantic import BaseModel, Field
from metagpt.actions import Action, ActionOutput
from metagpt.config import CONFIG
from metagpt.llm import LLM
from metagpt.llm import LLM, HumanProvider
from metagpt.logs import logger
from metagpt.memory import LongTermMemory, Memory
from metagpt.memory import Memory
# from metagpt.memory import LongTermMemory
from metagpt.schema import Message, MessageQueue
from metagpt.utils.common import any_to_str
@ -40,12 +43,14 @@ Please note that only the text between the first and second "===" is information
{history}
===
You can now choose one of the following stages to decide the stage you need to go in the next step:
Your previous stage: {previous_state}
Now choose one of the following stages you need to go to in the next step:
{states}
Just answer a number between 0-{n_states}, choose the most suitable stage according to the understanding of the conversation.
Please note that the answer only needs a number, no need to add any other text.
If there is no conversation record, choose 0.
If you think you have completed your goal and don't need to go to any of the stages, return -1.
Do not answer anything else, and do not add any other information in your answer.
"""
@ -60,6 +65,16 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi
"""
class RoleReactMode(str, Enum):
REACT = "react"
BY_ORDER = "by_order"
PLAN_AND_ACT = "plan_and_act"
@classmethod
def values(cls):
return [item.value for item in cls]
class RoleSetting(BaseModel):
"""Role Settings"""
@ -68,6 +83,7 @@ class RoleSetting(BaseModel):
goal: str
constraints: str
desc: str
is_human: bool
def __str__(self):
return f"{self.name}({self.profile})"
@ -82,11 +98,15 @@ class RoleContext(BaseModel):
env: "Environment" = Field(default=None)
msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates
memory: Memory = Field(default_factory=Memory)
long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory)
state: int = Field(default=0)
# long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory)
state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None
todo: Action = Field(default=None)
watch: set[str] = Field(default_factory=set)
news: list[Type[Message]] = Field(default=[])
react_mode: RoleReactMode = (
RoleReactMode.REACT
) # see `Role._set_react_mode` for definitions of the following two attributes
max_react_loop: int = 1
class Config:
arbitrary_types_allowed = True
@ -109,9 +129,11 @@ class RoleContext(BaseModel):
class Role:
"""Role/Agent"""
def __init__(self, name="", profile="", goal="", constraints="", desc=""):
self._llm = LLM()
self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc)
def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False):
self._llm = LLM() if not is_human else HumanProvider()
self._setting = RoleSetting(
name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human
)
self._states = []
self._actions = []
self._role_id = str(self._setting)
@ -126,13 +148,40 @@ class Role:
self._reset()
for idx, action in enumerate(actions):
if not isinstance(action, Action):
i = action("")
i = action("", llm=self._llm)
else:
if self._setting.is_human and not isinstance(action.llm, HumanProvider):
logger.warning(
f"is_human attribute does not take effect,"
f"as Role's {str(action)} was initialized using LLM, try passing in Action classes instead of initialized instances"
)
i = action
i.set_env(self._rc.env)
i.set_prefix(self._get_prefix(), self.profile)
self._actions.append(i)
self._states.append(f"{idx}. {action}")
def _set_react_mode(self, react_mode: str, max_react_loop: int = 1):
"""Set strategy of the Role reacting to observed Message. Variation lies in how
this Role elects action to perform during the _think stage, especially if it is capable of multiple Actions.
Args:
react_mode (str): Mode for choosing action during the _think stage, can be one of:
"react": standard think-act loop in the ReAct paper, alternating thinking and acting to solve the task, i.e. _think -> _act -> _think -> _act -> ...
Use llm to select actions in _think dynamically;
"by_order": switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ...;
"plan_and_act": first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ...
Use llm to come up with the plan dynamically.
Defaults to "react".
max_react_loop (int): Maximum react cycles to execute, used to prevent the agent from reacting forever.
Take effect only when react_mode is react, in which we use llm to choose actions, including termination.
Defaults to 1, i.e. _think -> _act (-> return result and end)
"""
assert react_mode in RoleReactMode.values(), f"react_mode must be one of {RoleReactMode.values()}"
self._rc.react_mode = react_mode
if react_mode == RoleReactMode.REACT:
self._rc.max_react_loop = max_react_loop
def _watch(self, actions: Iterable[Type[Action]]):
"""Watch Actions of interest. Role will select Messages caused by these Actions from its personal message
buffer during _observe.
@ -151,11 +200,11 @@ class Role:
if self._rc.env: # According to the routing feature plan in Chapter 2.2.3.2 of RFC 113
self._rc.env.set_subscription(self, self._subscription)
def _set_state(self, state):
def _set_state(self, state: int):
"""Update the current state."""
self._rc.state = state
logger.debug(self._actions)
self._rc.todo = self._actions[self._rc.state]
self._rc.todo = self._actions[self._rc.state] if state >= 0 else None
def set_env(self, env: "Environment"):
"""Set the environment in which the role works. The role can talk to the environment and can also receive
@ -164,6 +213,22 @@ class Role:
if env:
env.set_subscription(self, self._subscription)
# # Replaced by FileRepository.set_file
# def set_doc(self, content: str, filename: str):
# return self._rc.env.set_doc(content, filename)
#
# # Replaced by FileRepository.get_file
# def get_doc(self, filename: str):
# return self._rc.env.get_doc(filename)
#
# # Replaced by CONFIG.xx
# def set(self, k, v):
# return self._rc.env.set(k, v)
#
# # Replaced by CONFIG.xx
# def get(self, k):
# return self._rc.env.get(k)
@property
def profile(self):
"""Get the role description (position)"""
@ -193,14 +258,22 @@ class Role:
return
prompt = self._get_prefix()
prompt += STATE_TEMPLATE.format(
history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1
history=self._rc.history,
states="\n".join(self._states),
n_states=len(self._states) - 1,
previous_state=self._rc.state,
)
# print(prompt)
next_state = await self._llm.aask(prompt)
logger.debug(f"{prompt=}")
if not next_state.isdigit() or int(next_state) not in range(len(self._states)):
logger.warning(f"Invalid answer of state, {next_state=}")
next_state = "0"
self._set_state(int(next_state))
if (not next_state.isdigit() and next_state != "-1") or int(next_state) not in range(-1, len(self._states)):
logger.warning(f"Invalid answer of state, {next_state=}, will be set to -1")
next_state = -1
else:
next_state = int(next_state)
if next_state == -1:
logger.info(f"End actions with {next_state=}")
self._set_state(next_state)
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
@ -250,10 +323,66 @@ class Role:
self._rc.msg_buffer.push(message)
async def _react(self) -> Message:
"""Think first, then act"""
await self._think()
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
return await self._act()
"""Think first, then act, until the Role _think it is time to stop and requires no more todo.
This is the standard think-act loop in the ReAct paper, which alternates thinking and acting in task solving, i.e. _think -> _act -> _think -> _act -> ...
Use llm to select actions in _think dynamically
"""
actions_taken = 0
rsp = Message("No actions taken yet") # will be overwritten after Role _act
while actions_taken < self._rc.max_react_loop:
# think
await self._think()
if self._rc.todo is None:
break
# act
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
rsp = await self._act() # 这个rsp是否需要publish_message？
actions_taken += 1
return rsp # return output from the last action
async def _act_by_order(self) -> Message:
"""switch action each time by order defined in _init_actions, i.e. _act (Action1) -> _act (Action2) -> ..."""
for i in range(len(self._states)):
self._set_state(i)
rsp = await self._act()
return rsp # return output from the last action
async def _plan_and_act(self) -> Message:
"""first plan, then execute an action sequence, i.e. _think (of a plan) -> _act -> _act -> ... Use llm to come up with the plan dynamically."""
# TODO: to be implemented
return Message("")
async def react(self) -> Message:
"""Entry to one of three strategies by which Role reacts to the observed Message"""
if self._rc.react_mode == RoleReactMode.REACT:
rsp = await self._react()
elif self._rc.react_mode == RoleReactMode.BY_ORDER:
rsp = await self._act_by_order()
elif self._rc.react_mode == RoleReactMode.PLAN_AND_ACT:
rsp = await self._plan_and_act()
self._set_state(state=-1) # current reaction is complete, reset state to -1 and todo back to None
return rsp
# # Replaced by run()
# def recv(self, message: Message) -> None:
# """add message to history."""
# # self._history += f"\n{message}"
# # self._context = self._history
# if message in self._rc.memory.get():
# return
# self._rc.memory.add(message)
# # Replaced by run()
# async def handle(self, message: Message) -> Message:
# """Receive information and reply with actions"""
# # logger.debug(f"{self.name=}, {self.profile=}, {message.role=}")
# self.recv(message)
#
# return await self._react()
def get_memories(self, k=0) -> list[Message]:
"""A wrapper to return the most recent k memories of this role, return all when k=0"""
return self._rc.memory.get(k=k)
async def run(self, with_message=None):
"""Observe, and think and act based on the results of the observation"""

4
metagpt/roles/sk_agent.py
View file

@ -11,7 +11,7 @@ from semantic_kernel.planning import SequentialPlanner
from semantic_kernel.planning.action_planner.action_planner import ActionPlanner
from semantic_kernel.planning.basic_planner import BasicPlanner
from metagpt.actions import BossRequirement
from metagpt.actions import UserRequirement
from metagpt.actions.execute_task import ExecuteTask
from metagpt.logs import logger
from metagpt.roles import Role
@ -41,7 +41,7 @@ class SkAgent(Role):
"""Initializes the Engineer role with given attributes."""
super().__init__(name, profile, goal, constraints)
self._init_actions([ExecuteTask()])
self._watch([BossRequirement])
self._watch([UserRequirement])
self.kernel = make_sk_kernel()
# how funny the interface is inconsistent

21
metagpt/schema.py
View file

@ -18,6 +18,7 @@ import json
import os.path
from asyncio import Queue, QueueEmpty, wait_for
from json import JSONDecodeError
from pathlib import Path
from typing import Dict, List, Optional, Set, TypedDict
from pydantic import BaseModel, Field
@ -28,6 +29,8 @@ from metagpt.const import (
MESSAGE_ROUTE_FROM,
MESSAGE_ROUTE_TO,
MESSAGE_ROUTE_TO_ALL,
SYSTEM_DESIGN_FILE_REPO,
TASK_FILE_REPO,
)
from metagpt.logs import logger
from metagpt.utils.common import any_to_str, any_to_str_set
@ -312,3 +315,21 @@ class RunCodeResult(BaseModel):
return RunCodeResult(**m)
except Exception:
return None
class CodeSummarizeContext(BaseModel):
design_filename: str = ""
task_filename: str = ""
codes_filenames: Set[str] = Field(default_factory=set)
@staticmethod
def loads(filenames: Set) -> CodeSummarizeContext:
ctx = CodeSummarizeContext()
for filename in filenames:
if Path(filename).is_relative_to(SYSTEM_DESIGN_FILE_REPO):
ctx.design_filename = str(filename)
continue
if Path(filename).is_relative_to(TASK_FILE_REPO):
ctx.task_filename = str(filename)
continue
return ctx

52
metagpt/startup.py Normal file
View file

@ -0,0 +1,52 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import asyncio
import typer
app = typer.Typer()
@app.command()
def startup(
idea: str = typer.Argument(..., help="Your innovative idea, such as 'Create a 2048 game.'"),
investment: float = typer.Option(3.0, help="Dollar amount to invest in the AI company."),
n_round: int = typer.Option(5, help="Number of rounds for the simulation."),
code_review: bool = typer.Option(True, help="Whether to use code review."),
run_tests: bool = typer.Option(False, help="Whether to enable QA for adding & running tests."),
implement: bool = typer.Option(True, help="Enable or disable code implementation."),
project_name: str = typer.Option("", help="Unique project name, such as 'game_2048'."),
inc: bool = typer.Option(False, help="Incremental mode. Use it to coop with existing repo."),
):
"""Run a startup. Be a boss."""
from metagpt.roles import (
Architect,
Engineer,
ProductManager,
ProjectManager,
QaEngineer,
)
from metagpt.team import Team
company = Team()
company.hire(
[
ProductManager(),
Architect(),
ProjectManager(),
]
)
if implement or code_review:
company.hire([Engineer(n_borg=5, use_code_review=code_review)])
if run_tests:
company.hire([QaEngineer()])
company.invest(investment)
company.run_project(idea, project_name=project_name, inc=inc)
asyncio.run(company.run(n_round=n_round))
if __name__ == "__main__":
startup(idea="Make a 2048 game.")

30
metagpt/software_company.py → metagpt/team.py
View file

@ -9,7 +9,7 @@
"""
from pydantic import BaseModel, Field
from metagpt.actions import BossRequirement
from metagpt.actions import UserRequirement
from metagpt.config import CONFIG
from metagpt.environment import Environment
from metagpt.logs import logger
@ -18,13 +18,13 @@ from metagpt.schema import Message
from metagpt.utils.common import NoMoneyException
class SoftwareCompany(BaseModel):
class Team(BaseModel):
"""
Software Company: Possesses a team, SOP (Standard Operating Procedures), and a platform for instant messaging,
dedicated to writing executable code.
Team: Possesses one or more roles (agents), SOP (Standard Operating Procedures), and a platform for instant messaging,
dedicated to perform any multi-agent activity, such as collaboratively writing executable code.
"""
environment: Environment = Field(default_factory=Environment)
env: Environment = Field(default_factory=Environment)
investment: float = Field(default=10.0)
idea: str = Field(default="")
@ -33,7 +33,7 @@ class SoftwareCompany(BaseModel):
def hire(self, roles: list[Role]):
"""Hire roles to cooperate"""
self.environment.add_roles(roles)
self.env.add_roles(roles)
def invest(self, investment: float):
"""Invest company. raise NoMoneyException when exceed max_budget."""
@ -45,13 +45,19 @@ class SoftwareCompany(BaseModel):
if CONFIG.total_cost > CONFIG.max_budget:
raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}")
def start_project(self, idea):
"""Start a project from publishing boss requirement."""
def run_project(self, idea, send_to: str = "", project_name: str = "", inc: bool = False):
"""Start a project from publishing user requirement."""
self.idea = idea
self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement))
# If user set project_name, then use it.
if project_name:
path = CONFIG.workspace_path / project_name
self.env.load_existing_repo(path, inc=inc)
# Human requirement.
self.env.publish_message(Message(role="Human", content=idea, cause_by=UserRequirement, send_to=send_to))
def _save(self):
logger.info(self.json())
logger.info(self.json(ensure_ascii=False))
async def run(self, n_round=3):
"""Run company until target round or no money"""
@ -60,7 +66,7 @@ class SoftwareCompany(BaseModel):
n_round -= 1
logger.debug(f"{n_round=}")
self._check_balance()
await self.environment.run()
await self.env.run()
if CONFIG.git_repo:
CONFIG.git_repo.archive()
return self.environment.history
return self.env.history

14
metagpt/tools/sd_engine.py
View file

@ -13,11 +13,10 @@ from typing import List
from aiohttp import ClientSession
from PIL import Image, PngImagePlugin
from metagpt.config import Config
from metagpt.const import WORKSPACE_ROOT
from metagpt.logs import logger
from metagpt.config import CONFIG
config = Config()
# from metagpt.const import WORKSPACE_ROOT
from metagpt.logs import logger
payload = {
"prompt": "",
@ -56,9 +55,8 @@ default_negative_prompt = "(easynegative:0.8),black, dark,Low resolution"
class SDEngine:
def __init__(self):
# Initialize the SDEngine with configuration
self.config = Config()
self.sd_url = self.config.get("SD_URL")
self.sd_t2i_url = f"{self.sd_url}{self.config.get('SD_T2I_API')}"
self.sd_url = CONFIG.get("SD_URL")
self.sd_t2i_url = f"{self.sd_url}{CONFIG.get('SD_T2I_API')}"
# Define default payload settings for SD API
self.payload = payload
logger.info(self.sd_t2i_url)
@ -81,7 +79,7 @@ class SDEngine:
return self.payload
def _save(self, imgs, save_name=""):
save_dir = WORKSPACE_ROOT / "resources" / "SD_Output"
save_dir = CONFIG.workspace_path / "resources" / "SD_Output"
if not os.path.exists(save_dir):
os.makedirs(save_dir, exist_ok=True)
batch_decode_base64_to_image(imgs, save_dir, save_name=save_name)

2
metagpt/utils/common.py
View file

@ -9,6 +9,8 @@
@Modified By: mashenquan, 2023/11/27. Bug fix: `parse_recipient` failed to parse the recipient in certain GPT-3.5
responses.
"""
from __future__ import annotations
import ast
import contextlib
import inspect

10
metagpt/utils/file_repository.py
View file

@ -54,7 +54,7 @@ class FileRepository:
"""
pathname = self.workdir / filename
pathname.parent.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(str(pathname), mode="w") as writer:
async with aiofiles.open(str(pathname), mode="wb") as writer:
await writer.write(content)
logger.info(f"save to: {str(pathname)}")
@ -98,7 +98,7 @@ class FileRepository:
if not path_name.exists():
return None
try:
async with aiofiles.open(str(path_name), mode="r") as reader:
async with aiofiles.open(str(path_name), mode="rb") as reader:
doc.content = await reader.read()
except FileNotFoundError as e:
logger.info(f"open {str(path_name)} failed:{e}")
@ -178,7 +178,7 @@ class FileRepository:
# guid_suffix = str(uuid.uuid4())[:8]
# return f"{current_time}x{guid_suffix}"
async def save_doc(self, doc: Document, with_suffix:str = None, dependencies: List[str] = None):
async def save_doc(self, doc: Document, with_suffix: str = None, dependencies: List[str] = None):
"""Save a Document instance as a PDF file.
This method converts the content of the Document instance to Markdown,
@ -238,7 +238,9 @@ class FileRepository:
return await file_repo.save(filename=filename, content=content, dependencies=dependencies)
@staticmethod
async def save_as(doc:Document, with_suffix:str = None, dependencies: List[str] = None, relative_path: Path | str = "."):
async def save_as(
doc: Document, with_suffix: str = None, dependencies: List[str] = None, relative_path: Path | str = "."
):
"""Save a Document instance with optional modifications.
This static method creates a new FileRepository, saves the Document instance

13
metagpt/utils/mermaid.py
View file

@ -10,7 +10,7 @@ import os
from pathlib import Path
from metagpt.config import CONFIG
from metagpt.const import PROJECT_ROOT
from metagpt.const import METAGPT_ROOT
from metagpt.logs import logger
from metagpt.utils.common import check_cmd_exists
@ -34,7 +34,10 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
engine = CONFIG.mermaid_engine.lower()
if engine == "nodejs":
if check_cmd_exists(CONFIG.mmdc) != 0:
logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
logger.warning(
"RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc,"
"or consider changing MERMAID_ENGINE to `playwright`, `pyppeteer`, or `ink`."
)
return -1
for suffix in ["pdf", "svg", "png"]:
@ -66,7 +69,7 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
if stdout:
logger.info(stdout.decode())
if stderr:
logger.error(stderr.decode())
logger.warning(stderr.decode())
else:
if engine == "playwright":
from metagpt.utils.mmdc_playwright import mermaid_to_file
@ -138,6 +141,6 @@ MMC2 = """sequenceDiagram
if __name__ == "__main__":
loop = asyncio.new_event_loop()
result = loop.run_until_complete(mermaid_to_file(MMC1, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
result = loop.run_until_complete(mermaid_to_file(MMC2, PROJECT_ROOT / f"{CONFIG.mermaid_engine}/1"))
result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1"))
result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1"))
loop.close()

13
metagpt/utils/token_counter.py
View file

@ -21,7 +21,9 @@ TOKEN_COSTS = {
"gpt-4-32k": {"prompt": 0.06, "completion": 0.12},
"gpt-4-32k-0314": {"prompt": 0.06, "completion": 0.12},
"gpt-4-0613": {"prompt": 0.06, "completion": 0.12},
"gpt-4-1106-preview": {"prompt": 0.01, "completion": 0.03},
"text-embedding-ada-002": {"prompt": 0.0004, "completion": 0.0},
"chatglm_turbo": {"prompt": 0.0, "completion": 0.00069}, # 32k version, prompt + completion tokens=0.005￥/k-tokens
}
@ -36,7 +38,9 @@ TOKEN_MAX = {
"gpt-4-32k": 32768,
"gpt-4-32k-0314": 32768,
"gpt-4-0613": 8192,
"gpt-4-1106-preview": 128000,
"text-embedding-ada-002": 8192,
"chatglm_turbo": 32768,
}
@ -54,21 +58,24 @@ def count_message_tokens(messages, model="gpt-3.5-turbo-0613"):
"gpt-4-32k-0314",
"gpt-4-0613",
"gpt-4-32k-0613",
"gpt-4-1106-preview",
}:
tokens_per_message = 3
tokens_per_name = 1
elif model == "gpt-3.5-turbo-0301":
tokens_per_message = 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n
tokens_per_name = -1 # if there's a name, the role is omitted
elif "gpt-3.5-turbo" in model:
elif "gpt-3.5-turbo" == model:
print("Warning: gpt-3.5-turbo may update over time. Returning num tokens assuming gpt-3.5-turbo-0613.")
return count_message_tokens(messages, model="gpt-3.5-turbo-0613")
elif "gpt-4" in model:
elif "gpt-4" == model:
print("Warning: gpt-4 may update over time. Returning num tokens assuming gpt-4-0613.")
return count_message_tokens(messages, model="gpt-4-0613")
else:
raise NotImplementedError(
f"""num_tokens_from_messages() is not implemented for model {model}. See https://github.com/openai/openai-python/blob/main/chatml.md for information on how messages are converted to tokens."""
f"num_tokens_from_messages() is not implemented for model {model}. "
f"See https://github.com/openai/openai-python/blob/main/chatml.md "
f"for information on how messages are converted to tokens."
)
num_tokens = 0
for message in messages: