Merge pull request #1 from send18/dev

Dev
This commit is contained in:
Guess 2023-09-02 12:48:37 +08:00 committed by GitHub
commit 6b66429af8
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
24 changed files with 495 additions and 304 deletions

View file

@ -77,4 +77,10 @@ MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo
MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k
### Meta Models
#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL
#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL
### S3 config
S3:
access_key: "YOUR_S3_ACCESS_KEY"
secret_key: "YOUR_S3_SECRET_KEY"
endpoint_url: "YOUR_S3_ENDPOINT_URL"

View file

@ -7,6 +7,7 @@
@Modified By: mashenquan, 2023/8/20. Add function return annotations.
"""
from __future__ import annotations
from abc import ABC
from typing import Optional
@ -14,12 +15,12 @@ from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action_output import ActionOutput
from metagpt.llm import LLM
from metagpt.utils.common import OutputParser
from metagpt.logs import logger
from metagpt.utils.common import OutputParser
class Action(ABC):
def __init__(self, name: str = '', context=None, llm: LLM = None):
def __init__(self, name: str = "", context=None, llm: LLM = None):
self.name: str = name
if llm is None:
llm = LLM()
@ -50,9 +51,9 @@ class Action(ABC):
return await self.llm.aask(prompt, system_msgs)
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def _aask_v1(self, prompt: str, output_class_name: str,
output_data_mapping: dict,
system_msgs: Optional[list[str]] = None) -> ActionOutput:
async def _aask_v1(
self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None
) -> ActionOutput:
"""Append default prefix"""
if not system_msgs:
system_msgs = []

View file

@ -6,12 +6,12 @@
@File : design_api.py
@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class.
"""
import shutil
from pathlib import Path
from typing import List
from metagpt.actions import Action, ActionOutput
from metagpt.const import WORKSPACE_ROOT
import aiofiles
from metagpt.actions import Action
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.utils.common import CodeParser
from metagpt.utils.mermaid import mermaid_to_file
@ -93,52 +93,32 @@ OUTPUT_MAPPING = {
class WriteDesign(Action):
def __init__(self, name, context=None, llm=None):
super().__init__(name, context, llm)
self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \
"data structures, library tables, processes, and paths. Please provide your design, feedback " \
"clearly and in detail."
self.desc = (
"Based on the PRD, think about the system design, and design the corresponding APIs, "
"data structures, library tables, processes, and paths. Please provide your design, feedback "
"clearly and in detail."
)
def recreate_workspace(self, workspace: Path):
try:
shutil.rmtree(workspace)
except FileNotFoundError:
pass # 文件夹不存在,但我们不在意
workspace.mkdir(parents=True, exist_ok=True)
def _save_prd(self, docs_path, resources_path, prd):
prd_file = docs_path / 'prd.md'
quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd)
mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis')
logger.info(f"Saving PRD to {prd_file}")
prd_file.write_text(prd)
def _save_system_design(self, docs_path, resources_path, content):
async def _save_system_design(self, docs_path, resources_path, content):
data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content)
seq_flow = CodeParser.parse_code(block="Program call flow", text=content)
mermaid_to_file(data_api_design, resources_path / 'data_api_design')
mermaid_to_file(seq_flow, resources_path / 'seq_flow')
system_design_file = docs_path / 'system_design.md'
await mermaid_to_file(data_api_design, resources_path / "data_api_design")
await mermaid_to_file(seq_flow, resources_path / "seq_flow")
system_design_file = docs_path / "system_design.md"
logger.info(f"Saving System Designs to {system_design_file}")
system_design_file.write_text(content)
async with aiofiles.open(system_design_file, "w") as f:
await f.write(content)
def _save(self, context, system_design):
if isinstance(system_design, ActionOutput):
content = system_design.content
ws_name = CodeParser.parse_str(block="Python package name", text=content)
else:
content = system_design
ws_name = CodeParser.parse_str(block="Python package name", text=system_design)
workspace = WORKSPACE_ROOT / ws_name
self.recreate_workspace(workspace)
docs_path = workspace / 'docs'
resources_path = workspace / 'resources'
async def _save(self, system_design: str):
workspace = CONFIG.workspace
docs_path = workspace / "docs"
resources_path = workspace / "resources"
docs_path.mkdir(parents=True, exist_ok=True)
resources_path.mkdir(parents=True, exist_ok=True)
self._save_prd(docs_path, resources_path, context[-1].content)
self._save_system_design(docs_path, resources_path, content)
await self._save_system_design(docs_path, resources_path, system_design)
async def run(self, context, **kwargs):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
# system_design = await self._aask(prompt)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING)
self._save(context, system_design)
await self._save(system_design.content)
return system_design

View file

@ -8,11 +8,12 @@
"""
from typing import List, Tuple
from metagpt.actions.action import Action
from metagpt.const import WORKSPACE_ROOT
from metagpt.utils.common import CodeParser
import aiofiles
PROMPT_TEMPLATE = '''
from metagpt.actions.action import Action
from metagpt.config import CONFIG
PROMPT_TEMPLATE = """
# Context
{context}
@ -37,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
'''
"""
FORMAT_EXAMPLE = '''
---
@ -103,23 +104,24 @@ OUTPUT_MAPPING = {
class WriteTasks(Action):
def __init__(self, name="CreateTasks", context=None, llm=None):
super().__init__(name, context, llm)
def _save(self, context, rsp):
ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content)
file_path = WORKSPACE_ROOT / ws_name / 'docs/api_spec_and_tasks.md'
file_path.write_text(rsp.content)
async def _save(self, rsp):
file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md"
async with aiofiles.open(file_path, "w") as f:
await f.write(rsp.content)
# Write requirements.txt
requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt'
requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n'))
requirements_path = CONFIG.workspace / "requirements.txt"
async with aiofiles.open(requirements_path, "w") as f:
await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n'))
async def run(self, context, **kwargs):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING)
self._save(context, rsp)
await self._save(rsp)
return rsp

View file

@ -5,13 +5,12 @@
@Author : alexanderwu
@File : write_code.py
"""
from metagpt.actions import WriteDesign
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
from metagpt.const import WORKSPACE_ROOT
from metagpt.logs import logger
from metagpt.schema import Message
from metagpt.utils.common import CodeParser
from tenacity import retry, stop_after_attempt, wait_fixed
PROMPT_TEMPLATE = """
NOTICE
@ -49,23 +48,6 @@ class WriteCode(Action):
def _is_invalid(self, filename):
return any(i in filename for i in ["mp3", "wav"])
def _save(self, context, filename, code):
# logger.info(filename)
# logger.info(code_rsp)
if self._is_invalid(filename):
return
design = [i for i in context if i.cause_by == WriteDesign][0]
ws_name = CodeParser.parse_str(block="Python package name", text=design.content)
ws_path = WORKSPACE_ROOT / ws_name
if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]):
ws_path = ws_path / ws_name
code_path = ws_path / filename
code_path.parent.mkdir(parents=True, exist_ok=True)
code_path.write_text(code)
logger.info(f"Saving Code to {code_path}")
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def write_code(self, prompt):
code_rsp = await self._aask(prompt)
@ -74,7 +56,7 @@ class WriteCode(Action):
async def run(self, context, filename):
prompt = PROMPT_TEMPLATE.format(context=context, filename=filename)
logger.info(f'Writing {filename}..')
logger.info(f"Writing {filename}..")
code = await self.write_code(prompt)
# code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING)
# self._save(context, filename, code)

View file

@ -7,9 +7,14 @@
"""
from typing import List, Tuple
import aiofiles
from metagpt.actions import Action, ActionOutput
from metagpt.actions.search_and_summarize import SearchAndSummarize
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.utils.common import CodeParser
from metagpt.utils.mermaid import mermaid_to_file
PROMPT_TEMPLATE = """
# Context
@ -121,7 +126,7 @@ OUTPUT_MAPPING = {
"Competitive Quadrant Chart": (str, ...),
"Requirement Analysis": (str, ...),
"Requirement Pool": (List[Tuple[str, str]], ...),
"UI Design draft":(str, ...),
"UI Design draft": (str, ...),
"Anything UNCLEAR": (str, ...),
}
@ -139,8 +144,31 @@ class WritePRD(Action):
logger.info(sas.result)
logger.info(rsp)
prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info,
format_example=FORMAT_EXAMPLE)
prompt = PROMPT_TEMPLATE.format(
requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE
)
logger.debug(prompt)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
await self._save(prd.content)
return prd
async def _save_prd(self, docs_path, resources_path, prd):
prd_file = docs_path / "prd.md"
quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd)
await mermaid_to_file(
mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis"
)
async with aiofiles.open(prd_file, "w") as f:
await f.write(prd)
logger.info(f"Saving PRD to {prd_file}")
async def _save(self, prd):
workspace = CONFIG.workspace
workspace.mkdir(parents=True, exist_ok=True)
docs_path = workspace / "docs"
resources_path = workspace / "resources"
docs_path.mkdir(parents=True, exist_ok=True)
resources_path.mkdir(parents=True, exist_ok=True)
await self._save_prd(docs_path, resources_path, prd)

View file

@ -4,15 +4,17 @@
Provide configuration, singleton.
@Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`.
"""
import datetime
import json
import os
from copy import deepcopy
from typing import Any
from uuid import uuid4
import openai
import yaml
from metagpt.const import PROJECT_ROOT, OPTIONS
from metagpt.const import OPTIONS, PROJECT_ROOT, WORKSPACE_ROOT
from metagpt.logs import logger
from metagpt.tools import SearchEngineType, WebBrowserEngineType
from metagpt.utils.cost_manager import CostManager
@ -55,7 +57,7 @@ class Config(metaclass=Singleton):
self.openai_api_key = self._get("OPENAI_API_KEY")
self.anthropic_api_key = self._get("Anthropic_API_KEY")
if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and (
not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key
not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key
):
logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first")
self.openai_api_base = self._get("OPENAI_API_BASE")
@ -93,6 +95,11 @@ class Config(metaclass=Singleton):
self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY")
self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT")
workspace_uid = (
self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}"
)
self.workspace = WORKSPACE_ROOT / workspace_uid
def _init_with_config_files_and_env(self, yaml_file):
"""从config/key.yaml / config/config.yaml / env三处按优先级递减加载"""
configs = dict(os.environ)

View file

@ -14,9 +14,11 @@ def get_project_root():
"""逐级向上寻找项目根目录"""
current_path = Path.cwd()
while True:
if (current_path / '.git').exists() or \
(current_path / '.project_root').exists() or \
(current_path / '.gitignore').exists():
if (
(current_path / ".git").exists()
or (current_path / ".project_root").exists()
or (current_path / ".gitignore").exists()
):
return current_path
parent_path = current_path.parent
if parent_path == current_path:
@ -25,15 +27,15 @@ def get_project_root():
PROJECT_ROOT = get_project_root()
DATA_PATH = PROJECT_ROOT / 'data'
WORKSPACE_ROOT = PROJECT_ROOT / 'workspace'
PROMPT_PATH = PROJECT_ROOT / 'metagpt/prompts'
UT_PATH = PROJECT_ROOT / 'data/ut'
DATA_PATH = PROJECT_ROOT / "data"
WORKSPACE_ROOT = PROJECT_ROOT / "workspace"
PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts"
UT_PATH = PROJECT_ROOT / "data/ut"
SWAGGER_PATH = UT_PATH / "files/api/"
UT_PY_PATH = UT_PATH / "files/ut/"
API_QUESTIONS_PATH = UT_PATH / "files/question/"
YAPI_URL = "http://yapi.deepwisdomai.com/"
TMP = PROJECT_ROOT / 'tmp'
TMP = PROJECT_ROOT / "tmp"
RESEARCH_PATH = DATA_PATH / "research"
MEM_TTL = 24 * 30 * 3600
@ -43,4 +45,12 @@ DEFAULT_LANGUAGE = "English"
DEFAULT_MAX_TOKENS = 1500
COMMAND_TOKENS = 500
BRAIN_MEMORY = "BRAIN_MEMORY"
SKILL_PATH = "SKILL_PATH"
SKILL_PATH = "SKILL_PATH"
SERPER_API_KEY = "SERPER_API_KEY"
# Key Definitions for MetaGPT LLM
METAGPT_API_MODEL = "METAGPT_API_MODEL"
METAGPT_API_KEY = "METAGPT_API_KEY"
METAGPT_API_BASE = "METAGPT_API_BASE"
METAGPT_API_TYPE = "METAGPT_API_TYPE"
METAGPT_API_VERSION = "METAGPT_API_VERSION"

View file

@ -15,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot
class BaseGPTAPI(BaseChatbot):
"""GPT API abstract class, requiring all inheritors to provide a series of standard capabilities"""
system_prompt = 'You are a helpful assistant.'
system_prompt = "You are a helpful assistant."
def _user_msg(self, msg: str) -> dict[str, str]:
return {"role": "user", "content": msg}
@ -46,9 +47,9 @@ class BaseGPTAPI(BaseChatbot):
rsp = await self.acompletion_text(message, stream=True)
except Exception as e:
logger.exception(f"{e}")
logger.info(f"ask:{msg}, error:{e}")
raise e
logger.debug(message)
# logger.debug(rsp)
logger.info(f"ask:{msg}, anwser:{rsp}")
return rsp
def _extract_assistant_rsp(self, context):
@ -115,7 +116,7 @@ class BaseGPTAPI(BaseChatbot):
def messages_to_prompt(self, messages: list[dict]):
"""[{"role": "user", "content": msg}] to user: <msg> etc."""
return '\n'.join([f"{i['role']}: {i['content']}" for i in messages])
return "\n".join([f"{i['role']}: {i['content']}" for i in messages])
def messages_to_dict(self, messages):
"""objects to [{"role": "user", "content": msg}] etc."""

View file

@ -0,0 +1,33 @@
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/30
@Author : mashenquan
@File : metagpt_llm_api.py
@Desc : MetaGPT LLM related APIs
"""
import openai
from metagpt.config import CONFIG
from metagpt.provider import OpenAIGPTAPI
from metagpt.provider.openai_api import RateLimiter
class MetaGPTLLMAPI(OpenAIGPTAPI):
"""MetaGPT LLM api"""
def __init__(self):
self.__init_openai()
self.llm = openai
self.model = CONFIG.METAGPT_API_MODEL
self.auto_max_tokens = False
RateLimiter.__init__(self, rpm=self.rpm)
def __init_openai(self, *args, **kwargs):
openai.api_key = CONFIG.METAGPT_API_KEY
if CONFIG.METAGPT_API_BASE:
openai.api_base = CONFIG.METAGPT_API_BASE
if CONFIG.METAGPT_API_TYPE:
openai.api_type = CONFIG.METAGPT_API_TYPE
openai.api_version = CONFIG.METAGPT_API_VERSION
self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10

View file

@ -6,17 +6,18 @@
@File : engineer.py
"""
import asyncio
import shutil
from collections import OrderedDict
from pathlib import Path
from metagpt.const import WORKSPACE_ROOT
import aiofiles
from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign
from metagpt.schema import Message
from metagpt.utils.common import CodeParser
from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP
from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP
async def gather_ordered_k(coros, k) -> list:
@ -47,9 +48,15 @@ async def gather_ordered_k(coros, k) -> list:
class Engineer(Role):
def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code",
constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain",
n_borg=1, use_code_review=False):
def __init__(
self,
name="Alex",
profile="Engineer",
goal="Write elegant, readable, extensible, efficient code",
constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain",
n_borg=1,
use_code_review=False,
):
super().__init__(name, profile, goal, constraints)
self._init_actions([WriteCode])
self.use_code_review = use_code_review
@ -72,31 +79,24 @@ class Engineer(Role):
@classmethod
def parse_workspace(cls, system_design_msg: Message) -> str:
if system_design_msg.instruct_content:
return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip("\"")
return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"')
return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
def get_workspace(self) -> Path:
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
if not msg:
return WORKSPACE_ROOT / 'src'
return CONFIG.workspace / "src"
workspace = self.parse_workspace(msg)
# Codes are written in workspace/{package_name}/{package_name}
return WORKSPACE_ROOT / workspace / workspace
return CONFIG.workspace / workspace
def recreate_workspace(self):
async def write_file(self, filename: str, code: str):
workspace = self.get_workspace()
try:
shutil.rmtree(workspace)
except FileNotFoundError:
pass # 文件夹不存在,但我们不在意
workspace.mkdir(parents=True, exist_ok=True)
def write_file(self, filename: str, code: str):
workspace = self.get_workspace()
filename = filename.replace('"', '').replace('\n', '')
filename = filename.replace('"', "").replace("\n", "")
file = workspace / filename
file.parent.mkdir(parents=True, exist_ok=True)
file.write_text(code)
async with aiofiles.open(file, "w") as f:
await f.write(code)
return file
def recv(self, message: Message) -> None:
@ -109,8 +109,7 @@ class Engineer(Role):
todo_coros = []
for todo in self.todos:
todo_coro = WriteCode().run(
context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]),
filename=todo
context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo
)
todo_coros.append(todo_coro)
@ -124,38 +123,40 @@ class Engineer(Role):
self._rc.memory.add(msg)
del self.todos[0]
logger.info(f'Done {self.get_workspace()} generating.')
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo))
return msg
async def _act_sp(self) -> Message:
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
instruct_content = {}
for todo in self.todos:
code = await WriteCode().run(
context=self._rc.history,
filename=todo
)
code = await WriteCode().run(context=self._rc.history, filename=todo)
# logger.info(todo)
# logger.info(code_rsp)
# code = self.parse_code(code_rsp)
file_path = self.write_file(todo, code)
file_path = await self.write_file(todo, code)
msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo))
self._rc.memory.add(msg)
instruct_content[todo] = code
code_msg = todo + FILENAME_CODE_SEP + str(file_path)
# code_msg = todo + FILENAME_CODE_SEP + str(file_path)
code_msg = (todo, file_path)
code_msg_all.append(code_msg)
logger.info(f'Done {self.get_workspace()} generating.')
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(
content=MSG_SEP.join(code_msg_all),
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
instruct_content=instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
send_to="QaEngineer"
send_to="QaEngineer",
)
return msg
async def _act_sp_precision(self) -> Message:
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
instruct_content = {}
for todo in self.todos:
"""
# 从历史信息中挑选必须的信息以减少prompt长度人工经验总结
@ -170,35 +171,30 @@ class Engineer(Role):
context.append(m.content)
context_str = "\n".join(context)
# 编写code
code = await WriteCode().run(
context=context_str,
filename=todo
)
code = await WriteCode().run(context=context_str, filename=todo)
# code review
if self.use_code_review:
try:
rewrite_code = await WriteCodeReview().run(
context=context_str,
code=code,
filename=todo
)
rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo)
code = rewrite_code
except Exception as e:
logger.error("code review failed!", e)
pass
file_path = self.write_file(todo, code)
file_path = await self.write_file(todo, code)
msg = Message(content=code, role=self.profile, cause_by=WriteCode)
self._rc.memory.add(msg)
instruct_content[todo] = code
code_msg = todo + FILENAME_CODE_SEP + str(file_path)
code_msg = (todo, file_path)
code_msg_all.append(code_msg)
logger.info(f'Done {self.get_workspace()} generating.')
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(
content=MSG_SEP.join(code_msg_all),
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
instruct_content=instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
send_to="QaEngineer"
send_to="QaEngineer",
)
return msg

View file

@ -9,7 +9,7 @@ import os
from pathlib import Path
from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest
from metagpt.const import WORKSPACE_ROOT
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
@ -43,13 +43,13 @@ class QaEngineer(Role):
def get_workspace(self, return_proj_dir=True) -> Path:
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
if not msg:
return WORKSPACE_ROOT / "src"
return CONFIG.workspace / "src"
workspace = self.parse_workspace(msg)
# project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc.
if return_proj_dir:
return WORKSPACE_ROOT / workspace
return CONFIG.workspace / workspace
# development codes directory: workspace/{package_name}/{package_name}
return WORKSPACE_ROOT / workspace / workspace
return CONFIG.workspace / workspace / workspace
def write_file(self, filename: str, code: str):
workspace = self.get_workspace() / "tests"

View file

@ -11,15 +11,14 @@ from __future__ import annotations
from typing import Iterable, Type
from pydantic import BaseModel, Field
from metagpt.actions import Action, ActionOutput
from metagpt.config import CONFIG
from metagpt.const import OPTIONS
from metagpt.llm import LLM
from metagpt.actions import Action, ActionOutput
from metagpt.logs import logger
from metagpt.memory import Memory, LongTermMemory
from metagpt.memory import LongTermMemory, Memory
from metagpt.schema import Message, MessageTag
PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """
@ -52,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi
class RoleSetting(BaseModel):
"""Role properties"""
name: str
profile: str
goal: str
@ -67,7 +67,8 @@ class RoleSetting(BaseModel):
class RoleContext(BaseModel):
"""Runtime role context"""
env: 'Environment' = Field(default=None)
env: "Environment" = Field(default=None)
memory: Memory = Field(default_factory=Memory)
long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory)
state: int = Field(default=0)
@ -95,7 +96,7 @@ class RoleContext(BaseModel):
@property
def prerequisite(self):
"""Retrieve information with `prerequisite` tag"""
if self.memory and hasattr(self.memory, 'get_by_tags'):
if self.memory and hasattr(self.memory, "get_by_tags"):
return self.memory.get_by_tags([MessageTag.Prerequisite.value])
return ""
@ -145,7 +146,7 @@ class Role:
logger.debug(self._actions)
self._rc.todo = self._actions[self._rc.state]
def set_env(self, env: 'Environment'):
def set_env(self, env: "Environment"):
"""设置角色工作所处的环境,角色可以向环境说话,也可以通过观察接受环境消息"""
self._rc.env = env
@ -192,12 +193,13 @@ class Role:
self._set_state(0)
return True
prompt = self._get_prefix()
prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states),
n_states=len(self._states) - 1)
prompt += STATE_TEMPLATE.format(
history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1
)
next_state = await self._llm.aask(prompt)
logger.debug(f"{prompt=}")
if not next_state.isdigit() or int(next_state) not in range(len(self._states)):
logger.warning(f'Invalid answer of state, {next_state=}')
logger.warning(f"Invalid answer of state, {next_state=}")
next_state = "0"
self._set_state(int(next_state))
return True
@ -212,8 +214,12 @@ class Role:
response = await self._rc.todo.run(requirement)
# logger.info(response)
if isinstance(response, ActionOutput):
msg = Message(content=response.content, instruct_content=response.instruct_content,
role=self.profile, cause_by=type(self._rc.todo))
msg = Message(
content=response.content,
instruct_content=response.instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
)
else:
msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo))
self._rc.memory.add(msg)
@ -236,7 +242,7 @@ class Role:
news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news]
if news_text:
logger.debug(f'{self._setting} observed: {news_text}')
logger.debug(f"{self._setting} observed: {news_text}")
return len(self._rc.news)
def _publish_message(self, msg):
@ -310,20 +316,15 @@ class Role:
def add_to_do(self, act):
self._rc.todo = act
async def think(self) -> bool:
async def think(self) -> Action:
"""The exported `think` function"""
has_action = await self._think()
if not has_action:
return False
if not self._rc.todo:
return False
return True
await self._think()
return self._rc.todo
async def act(self) -> ActionOutput:
"""The exported `act` function"""
msg = await self._act()
return ActionOutput(content=msg.content,
instruct_content=msg.instruct_content)
return ActionOutput(content=msg.content, instruct_content=msg.instruct_content)
@property
def todo_description(self):

View file

@ -9,22 +9,34 @@
"""
import re
import aiofiles
from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement
from metagpt.const import WORKSPACE_ROOT
from metagpt.actions.write_teaching_plan import (
TeachingPlanRequirement,
WriteTeachingPlanPart,
)
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
from metagpt.logs import logger
import re
class Teacher(Role):
"""Support configurable teacher roles,
with native and teaching languages being replaceable through configurations."""
def __init__(self, name='Lily', profile='{teaching_language} Teacher',
goal='writing a {language} teaching plan part by part',
constraints='writing in {language}', desc="", *args, **kwargs):
def __init__(
self,
name="Lily",
profile="{teaching_language} Teacher",
goal="writing a {language} teaching plan part by part",
constraints="writing in {language}",
desc="",
*args,
**kwargs,
):
super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs)
actions = []
for topic in WriteTeachingPlanPart.TOPICS:
@ -54,7 +66,7 @@ class Teacher(Role):
break
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
msg = await self._act()
if ret.content != '':
if ret.content != "":
ret.content += "\n\n\n"
ret.content += msg.content
logger.info(ret.content)
@ -64,14 +76,14 @@ class Teacher(Role):
async def save(self, content):
"""Save teaching plan"""
filename = Teacher.new_file_name(self.course_title)
pathname = WORKSPACE_ROOT / "teaching_plan"
pathname = CONFIG.workspace / "teaching_plan"
pathname.mkdir(exist_ok=True)
pathname = pathname / filename
try:
async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer:
async with aiofiles.open(str(pathname), mode="w", encoding="utf-8") as writer:
await writer.write(content)
except Exception as e:
logger.error(f'Save failed{e}')
logger.error(f"Save failed{e}")
logger.info(f"Save to:{pathname}")
@staticmethod
@ -80,8 +92,8 @@ class Teacher(Role):
# Define the special characters that need to be replaced.
illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']'
# Replace the special characters with underscores.
filename = re.sub(illegal_chars, '_', lesson_title) + ext
return re.sub(r'_+', '_', filename)
filename = re.sub(illegal_chars, "_", lesson_title) + ext
return re.sub(r"_+", "_", filename)
@property
def course_title(self):
@ -93,9 +105,9 @@ class Teacher(Role):
if act.rsp is None:
return default_title
title = act.rsp.lstrip("# \n")
if '\n' in title:
ix = title.index('\n')
title = title[0: ix]
if "\n" in title:
ix = title.index("\n")
title = title[0:ix]
return title
return default_title

View file

@ -14,7 +14,6 @@ from aiohttp import ClientSession
from PIL import Image, PngImagePlugin
from metagpt.config import Config
from metagpt.const import WORKSPACE_ROOT
from metagpt.logs import logger
config = Config()
@ -81,7 +80,7 @@ class SDEngine:
return self.payload
def _save(self, imgs, save_name=""):
save_dir = WORKSPACE_ROOT / "resources" / "SD_Output"
save_dir = CONFIG.get_workspace() / "resources" / "SD_Output"
if not os.path.exists(save_dir):
os.makedirs(save_dir, exist_ok=True)
batch_decode_base64_to_image(imgs, save_dir, save_name=save_name)

View file

@ -8,11 +8,11 @@ from __future__ import annotations
import asyncio
import sys
from pathlib import Path
from typing import Literal, Dict
from typing import Literal
from playwright.async_api import async_playwright
from metagpt.config import Config
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.utils.parse_html import WebPage
@ -28,20 +28,18 @@ class PlaywrightWrapper:
def __init__(
self,
options: Dict,
browser_type: Literal["chromium", "firefox", "webkit"] | None = None,
launch_kwargs: dict | None = None,
**kwargs,
) -> None:
self.options = options
if browser_type is None:
browser_type = options.get("playwright_browser_type")
browser_type = CONFIG.playwright_browser_type
self.browser_type = browser_type
launch_kwargs = launch_kwargs or {}
if options.get("global_proxy") and "proxy" not in launch_kwargs:
if CONFIG.global_proxy and "proxy" not in launch_kwargs:
args = launch_kwargs.get("args", [])
if not any(str.startswith(i, "--proxy-server=") for i in args):
launch_kwargs["proxy"] = {"server": options.get("global_proxy")}
launch_kwargs["proxy"] = {"server": CONFIG.global_proxy}
self.launch_kwargs = launch_kwargs
context_kwargs = {}
if "ignore_https_errors" in kwargs:
@ -81,8 +79,8 @@ class PlaywrightWrapper:
executable_path = Path(browser_type.executable_path)
if not executable_path.exists() and "executable_path" not in self.launch_kwargs:
kwargs = {}
if self.options.get("global_proxy"):
kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")}
if CONFIG.global_proxy:
kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy}
await _install_browsers(self.browser_type, **kwargs)
if self._has_run_precheck:
@ -150,8 +148,6 @@ if __name__ == "__main__":
import fire
async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs):
return await PlaywrightWrapper(options=Config().runtime_options,
browser_type=browser_type,
**kwargs).run(url, *urls)
return await PlaywrightWrapper(browser_type=browser_type, **kwargs).run(url, *urls)
fire.Fire(main)

View file

@ -6,19 +6,20 @@
@File : mermaid.py
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
import subprocess
import asyncio
from pathlib import Path
from metagpt.config import Config
# from metagpt.utils.common import check_cmd_exists
import aiofiles
from metagpt.config import CONFIG, Config
from metagpt.const import PROJECT_ROOT
from metagpt.logs import logger
from metagpt.utils.common import check_cmd_exists
def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
"""suffix: png/svg/pdf
:param options: runtime context options, created by `Config` class object and changed in flow pipeline
:param mermaid_code: mermaid code
:param output_file_without_suffix: output filename
:param width:
@ -27,92 +28,87 @@ def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=204
"""
# Write the Mermaid code to a temporary file
tmp = Path(f"{output_file_without_suffix}.mmd")
tmp.write_text(mermaid_code, encoding="utf-8")
async with aiofiles.open(tmp, "w", encoding="utf-8") as f:
await f.write(mermaid_code)
# tmp.write_text(mermaid_code, encoding="utf-8")
if check_cmd_exists("mmdc") != 0:
logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
return -1
# if check_cmd_exists("mmdc") != 0:
# logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
# return -1
for suffix in ["pdf", "svg", "png"]:
# for suffix in ["pdf", "svg", "png"]:
for suffix in ["png"]:
output_file = f"{output_file_without_suffix}.{suffix}"
# Call the `mmdc` command to convert the Mermaid code to a PNG
logger.info(f"Generating {output_file}..")
cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
if options.get("puppeteer_config"):
subprocess.run(
[
options.get("mmdc"),
"-p",
options.get("puppeteer_config"),
"-i",
str(tmp),
"-o",
output_file,
"-w",
str(width),
"-H",
str(height),
]
)
else:
subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)])
return 0
MMC1 = """classDiagram
class Main {
-SearchEngine search_engine
+main() str
}
class SearchEngine {
-Index index
-Ranking ranking
-Summary summary
+search(query: str) str
}
class Index {
-KnowledgeBase knowledge_base
+create_index(data: dict)
+query_index(query: str) list
}
class Ranking {
+rank_results(results: list) list
}
class Summary {
+summarize_results(results: list) str
}
class KnowledgeBase {
+update(data: dict)
+fetch_data(query: str) dict
}
Main --> SearchEngine
SearchEngine --> Index
SearchEngine --> Ranking
SearchEngine --> Summary
Index --> KnowledgeBase"""
MMC2 = """sequenceDiagram
participant M as Main
participant SE as SearchEngine
participant I as Index
participant R as Ranking
participant S as Summary
participant KB as KnowledgeBase
M->>SE: search(query)
SE->>I: query_index(query)
I->>KB: fetch_data(query)
KB-->>I: return data
I-->>SE: return results
SE->>R: rank_results(results)
R-->>SE: return ranked_results
SE->>S: summarize_results(ranked_results)
S-->>SE: return summary
SE-->>M: return summary"""
if CONFIG.puppeteer_config:
cmds.extend(["-p", CONFIG.puppeteer_config])
process = await asyncio.create_subprocess_exec(*cmds)
await process.wait()
return process.returncode
if __name__ == "__main__":
MMC1 = """classDiagram
class Main {
-SearchEngine search_engine
+main() str
}
class SearchEngine {
-Index index
-Ranking ranking
-Summary summary
+search(query: str) str
}
class Index {
-KnowledgeBase knowledge_base
+create_index(data: dict)
+query_index(query: str) list
}
class Ranking {
+rank_results(results: list) list
}
class Summary {
+summarize_results(results: list) str
}
class KnowledgeBase {
+update(data: dict)
+fetch_data(query: str) dict
}
Main --> SearchEngine
SearchEngine --> Index
SearchEngine --> Ranking
SearchEngine --> Summary
Index --> KnowledgeBase"""
MMC2 = """sequenceDiagram
participant M as Main
participant SE as SearchEngine
participant I as Index
participant R as Ranking
participant S as Summary
participant KB as KnowledgeBase
M->>SE: search(query)
SE->>I: query_index(query)
I->>KB: fetch_data(query)
KB-->>I: return data
I-->>SE: return results
SE->>R: rank_results(results)
R-->>SE: return ranked_results
SE->>S: summarize_results(ranked_results)
S-->>SE: return summary
SE-->>M: return summary"""
conf = Config()
mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1,
output_file_without_suffix=PROJECT_ROOT / "tmp/1.png")
mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2,
output_file_without_suffix=PROJECT_ROOT / "tmp/2.png")
asyncio.run(
mermaid_to_file(
options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png"
)
)
asyncio.run(
mermaid_to_file(
options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png"
)
)

127
metagpt/utils/s3.py Normal file
View file

@ -0,0 +1,127 @@
from typing import Optional
import aioboto3
from metagpt.logs import logger
from metagpt.config import Config
class S3:
"""A class for interacting with Amazon S3 storage."""
def __init__(self):
self.session = aioboto3.Session()
self.s3_config = Config().get("S3")
self.auth_config = {
"service_name": "s3",
"aws_access_key_id": self.s3_config["access_key"],
"aws_secret_access_key": self.s3_config["secret_key"],
"endpoint_url": self.s3_config["endpoint_url"]
}
async def upload_file(
self,
bucket: str,
local_path: str,
object_name: str,
) -> None:
"""Upload a file from the local path to the specified path of the storage bucket specified in s3.
Args:
bucket: The name of the S3 storage bucket.
local_path: The local file path, including the file name.
object_name: The complete path of the uploaded file to be stored in S3, including the file name.
Raises:
Exception: If an error occurs during the upload process, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
with open(local_path, "rb") as file:
await client.put_object(Body=file, Bucket=bucket, Key=object_name)
logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.")
except Exception as e:
logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}")
raise e
async def get_object_url(
self,
bucket: str,
object_name: str,
) -> str:
"""Get the URL for a downloadable or preview file stored in the specified S3 bucket.
Args:
bucket: The name of the S3 storage bucket.
object_name: The complete path of the file stored in S3, including the file name.
Returns:
The URL for the downloadable or preview file.
Raises:
Exception: If an error occurs while retrieving the URL, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
file = await client.get_object(Bucket=bucket, Key=object_name)
return str(file["Body"].url)
except Exception as e:
logger.error(f"Failed to get the url for a downloadable or preview file: {e}")
raise e
async def get_object(
self,
bucket: str,
object_name: str,
) -> bytes:
"""Get the binary data of a file stored in the specified S3 bucket.
Args:
bucket: The name of the S3 storage bucket.
object_name: The complete path of the file stored in S3, including the file name.
Returns:
The binary data of the requested file.
Raises:
Exception: If an error occurs while retrieving the file data, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
s3_object = await client.get_object(Bucket=bucket, Key=object_name)
return await s3_object["Body"].read()
except Exception as e:
logger.error(f"Failed to get the binary data of the file: {e}")
raise e
async def download_file(
self,
bucket: str,
object_name: str,
local_path: str,
chunk_size: Optional[int] = 128 * 1024
) -> None:
"""Download an S3 object to a local file.
Args:
bucket: The name of the S3 storage bucket.
object_name: The complete path of the file stored in S3, including the file name.
local_path: The local file path where the S3 object will be downloaded.
chunk_size: The size of data chunks to read and write at a time. Default is 128 KB.
Raises:
Exception: If an error occurs during the download process, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
s3_object = await client.get_object(Bucket=bucket, Key=object_name)
stream = s3_object["Body"]
with open(local_path, 'wb') as local_file:
while True:
file_data = await stream.read(chunk_size)
if not file_data:
break
local_file.write(file_data)
except Exception as e:
logger.error(f"Failed to download the file from S3: {e}")
raise e

View file

@ -40,4 +40,5 @@ libcst==1.0.1
qdrant-client==1.4.0
connexion[swagger-ui]
aiohttp_jinja2
azure-cognitiveservices-speech==1.31.0
azure-cognitiveservices-speech==1.31.0
aioboto3~=11.3.0

View file

@ -9,7 +9,6 @@
from unittest.mock import Mock
import pytest
import pytest_asyncio
from metagpt.config import Config
from metagpt.logs import logger
@ -74,3 +73,4 @@ def proxy():
@pytest.fixture(scope="session", autouse=True)
def init_config():
Config()

View file

@ -0,0 +1,17 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/30
@Author : mashenquan
@File : test_metagpt_llm_api.py
"""
from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI
def test_metagpt():
llm = MetaGPTLLMAPI()
assert llm
if __name__ == "__main__":
test_metagpt()

View file

@ -8,7 +8,7 @@ from functools import wraps
from importlib import import_module
from metagpt.actions import Action, ActionOutput, WritePRD
from metagpt.const import WORKSPACE_ROOT
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
@ -214,7 +214,7 @@ class UIDesign(Action):
logger.info("Finish icon design using StableDiffusion API")
async def _save(self, css_content, html_content):
save_dir = WORKSPACE_ROOT / "resources" / "codes"
save_dir = CONFIG.workspace / "resources" / "codes"
if not os.path.exists(save_dir):
os.makedirs(save_dir, exist_ok=True)
# Save CSS and HTML content to files

View file

@ -8,11 +8,8 @@
@Modified By: mashenquan, 2023-8-17, move to `tools` folder.
"""
import asyncio
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt'
from metagpt.const import WORKSPACE_ROOT
from metagpt.config import CONFIG
from metagpt.tools.azure_tts import AzureTTS
@ -28,15 +25,13 @@ def test_azure_tts():
Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.”
</mstts:express-as>
"""
path = WORKSPACE_ROOT / "tts"
path = CONFIG.workspace / "tts"
path.mkdir(exist_ok=True, parents=True)
filename = path / "girl.wav"
loop = asyncio.new_event_loop()
v = loop.create_task(azure_tts.synthesize_speech(
lang="zh-CN",
voice="zh-CN-XiaomoNeural",
text=text,
output_file=str(filename)))
v = loop.create_task(
azure_tts.synthesize_speech(lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, output_file=str(filename))
)
result = loop.run_until_complete(v)
print(result)
@ -45,5 +40,5 @@ def test_azure_tts():
# TODO: 这里如果要检验还要额外加上对应的asr才能确保前后生成是接近一致的但现在还没有
if __name__ == '__main__':
if __name__ == "__main__":
test_azure_tts()

View file

@ -4,7 +4,8 @@
#
import os
from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT
from metagpt.config import CONFIG
from metagpt.tools.sd_engine import SDEngine
def test_sd_engine_init():
@ -21,5 +22,5 @@ def test_sd_engine_generate_prompt():
async def test_sd_engine_run_t2i():
sd_engine = SDEngine()
await sd_engine.run_t2i(prompts=["test"])
img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png"
img_path = CONFIG.workspace / "resources" / "SD_Output" / "output_0.png"
assert os.path.exists(img_path) == True