mirror of
https://github.com/FoundationAgents/MetaGPT.git
synced 2026-06-05 14:55:18 +02:00
commit
6b66429af8
24 changed files with 495 additions and 304 deletions
|
|
@ -77,4 +77,10 @@ MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo
|
|||
MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k
|
||||
|
||||
### Meta Models
|
||||
#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL
|
||||
#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL
|
||||
|
||||
### S3 config
|
||||
S3:
|
||||
access_key: "YOUR_S3_ACCESS_KEY"
|
||||
secret_key: "YOUR_S3_SECRET_KEY"
|
||||
endpoint_url: "YOUR_S3_ENDPOINT_URL"
|
||||
|
|
@ -7,6 +7,7 @@
|
|||
@Modified By: mashenquan, 2023/8/20. Add function return annotations.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from abc import ABC
|
||||
from typing import Optional
|
||||
|
||||
|
|
@ -14,12 +15,12 @@ from tenacity import retry, stop_after_attempt, wait_fixed
|
|||
|
||||
from metagpt.actions.action_output import ActionOutput
|
||||
from metagpt.llm import LLM
|
||||
from metagpt.utils.common import OutputParser
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.common import OutputParser
|
||||
|
||||
|
||||
class Action(ABC):
|
||||
def __init__(self, name: str = '', context=None, llm: LLM = None):
|
||||
def __init__(self, name: str = "", context=None, llm: LLM = None):
|
||||
self.name: str = name
|
||||
if llm is None:
|
||||
llm = LLM()
|
||||
|
|
@ -50,9 +51,9 @@ class Action(ABC):
|
|||
return await self.llm.aask(prompt, system_msgs)
|
||||
|
||||
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
|
||||
async def _aask_v1(self, prompt: str, output_class_name: str,
|
||||
output_data_mapping: dict,
|
||||
system_msgs: Optional[list[str]] = None) -> ActionOutput:
|
||||
async def _aask_v1(
|
||||
self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None
|
||||
) -> ActionOutput:
|
||||
"""Append default prefix"""
|
||||
if not system_msgs:
|
||||
system_msgs = []
|
||||
|
|
|
|||
|
|
@ -6,12 +6,12 @@
|
|||
@File : design_api.py
|
||||
@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class.
|
||||
"""
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
from metagpt.actions import Action, ActionOutput
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
import aiofiles
|
||||
|
||||
from metagpt.actions import Action
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.common import CodeParser
|
||||
from metagpt.utils.mermaid import mermaid_to_file
|
||||
|
|
@ -93,52 +93,32 @@ OUTPUT_MAPPING = {
|
|||
class WriteDesign(Action):
|
||||
def __init__(self, name, context=None, llm=None):
|
||||
super().__init__(name, context, llm)
|
||||
self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \
|
||||
"data structures, library tables, processes, and paths. Please provide your design, feedback " \
|
||||
"clearly and in detail."
|
||||
self.desc = (
|
||||
"Based on the PRD, think about the system design, and design the corresponding APIs, "
|
||||
"data structures, library tables, processes, and paths. Please provide your design, feedback "
|
||||
"clearly and in detail."
|
||||
)
|
||||
|
||||
def recreate_workspace(self, workspace: Path):
|
||||
try:
|
||||
shutil.rmtree(workspace)
|
||||
except FileNotFoundError:
|
||||
pass # 文件夹不存在,但我们不在意
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _save_prd(self, docs_path, resources_path, prd):
|
||||
prd_file = docs_path / 'prd.md'
|
||||
quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd)
|
||||
mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis')
|
||||
logger.info(f"Saving PRD to {prd_file}")
|
||||
prd_file.write_text(prd)
|
||||
|
||||
def _save_system_design(self, docs_path, resources_path, content):
|
||||
async def _save_system_design(self, docs_path, resources_path, content):
|
||||
data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content)
|
||||
seq_flow = CodeParser.parse_code(block="Program call flow", text=content)
|
||||
mermaid_to_file(data_api_design, resources_path / 'data_api_design')
|
||||
mermaid_to_file(seq_flow, resources_path / 'seq_flow')
|
||||
system_design_file = docs_path / 'system_design.md'
|
||||
await mermaid_to_file(data_api_design, resources_path / "data_api_design")
|
||||
await mermaid_to_file(seq_flow, resources_path / "seq_flow")
|
||||
system_design_file = docs_path / "system_design.md"
|
||||
logger.info(f"Saving System Designs to {system_design_file}")
|
||||
system_design_file.write_text(content)
|
||||
async with aiofiles.open(system_design_file, "w") as f:
|
||||
await f.write(content)
|
||||
|
||||
def _save(self, context, system_design):
|
||||
if isinstance(system_design, ActionOutput):
|
||||
content = system_design.content
|
||||
ws_name = CodeParser.parse_str(block="Python package name", text=content)
|
||||
else:
|
||||
content = system_design
|
||||
ws_name = CodeParser.parse_str(block="Python package name", text=system_design)
|
||||
workspace = WORKSPACE_ROOT / ws_name
|
||||
self.recreate_workspace(workspace)
|
||||
docs_path = workspace / 'docs'
|
||||
resources_path = workspace / 'resources'
|
||||
async def _save(self, system_design: str):
|
||||
workspace = CONFIG.workspace
|
||||
docs_path = workspace / "docs"
|
||||
resources_path = workspace / "resources"
|
||||
docs_path.mkdir(parents=True, exist_ok=True)
|
||||
resources_path.mkdir(parents=True, exist_ok=True)
|
||||
self._save_prd(docs_path, resources_path, context[-1].content)
|
||||
self._save_system_design(docs_path, resources_path, content)
|
||||
await self._save_system_design(docs_path, resources_path, system_design)
|
||||
|
||||
async def run(self, context, **kwargs):
|
||||
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
|
||||
# system_design = await self._aask(prompt)
|
||||
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING)
|
||||
self._save(context, system_design)
|
||||
await self._save(system_design.content)
|
||||
return system_design
|
||||
|
|
|
|||
|
|
@ -8,11 +8,12 @@
|
|||
"""
|
||||
from typing import List, Tuple
|
||||
|
||||
from metagpt.actions.action import Action
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.utils.common import CodeParser
|
||||
import aiofiles
|
||||
|
||||
PROMPT_TEMPLATE = '''
|
||||
from metagpt.actions.action import Action
|
||||
from metagpt.config import CONFIG
|
||||
|
||||
PROMPT_TEMPLATE = """
|
||||
# Context
|
||||
{context}
|
||||
|
||||
|
|
@ -37,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
|
|||
|
||||
## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs.
|
||||
|
||||
'''
|
||||
"""
|
||||
|
||||
FORMAT_EXAMPLE = '''
|
||||
---
|
||||
|
|
@ -103,23 +104,24 @@ OUTPUT_MAPPING = {
|
|||
|
||||
|
||||
class WriteTasks(Action):
|
||||
|
||||
def __init__(self, name="CreateTasks", context=None, llm=None):
|
||||
super().__init__(name, context, llm)
|
||||
|
||||
def _save(self, context, rsp):
|
||||
ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content)
|
||||
file_path = WORKSPACE_ROOT / ws_name / 'docs/api_spec_and_tasks.md'
|
||||
file_path.write_text(rsp.content)
|
||||
async def _save(self, rsp):
|
||||
file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md"
|
||||
async with aiofiles.open(file_path, "w") as f:
|
||||
await f.write(rsp.content)
|
||||
|
||||
# Write requirements.txt
|
||||
requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt'
|
||||
requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n'))
|
||||
requirements_path = CONFIG.workspace / "requirements.txt"
|
||||
|
||||
async with aiofiles.open(requirements_path, "w") as f:
|
||||
await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n'))
|
||||
|
||||
async def run(self, context, **kwargs):
|
||||
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
|
||||
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING)
|
||||
self._save(context, rsp)
|
||||
await self._save(rsp)
|
||||
return rsp
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -5,13 +5,12 @@
|
|||
@Author : alexanderwu
|
||||
@File : write_code.py
|
||||
"""
|
||||
from metagpt.actions import WriteDesign
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
|
||||
from metagpt.actions.action import Action
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.logs import logger
|
||||
from metagpt.schema import Message
|
||||
from metagpt.utils.common import CodeParser
|
||||
from tenacity import retry, stop_after_attempt, wait_fixed
|
||||
|
||||
PROMPT_TEMPLATE = """
|
||||
NOTICE
|
||||
|
|
@ -49,23 +48,6 @@ class WriteCode(Action):
|
|||
def _is_invalid(self, filename):
|
||||
return any(i in filename for i in ["mp3", "wav"])
|
||||
|
||||
def _save(self, context, filename, code):
|
||||
# logger.info(filename)
|
||||
# logger.info(code_rsp)
|
||||
if self._is_invalid(filename):
|
||||
return
|
||||
|
||||
design = [i for i in context if i.cause_by == WriteDesign][0]
|
||||
|
||||
ws_name = CodeParser.parse_str(block="Python package name", text=design.content)
|
||||
ws_path = WORKSPACE_ROOT / ws_name
|
||||
if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]):
|
||||
ws_path = ws_path / ws_name
|
||||
code_path = ws_path / filename
|
||||
code_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
code_path.write_text(code)
|
||||
logger.info(f"Saving Code to {code_path}")
|
||||
|
||||
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
|
||||
async def write_code(self, prompt):
|
||||
code_rsp = await self._aask(prompt)
|
||||
|
|
@ -74,7 +56,7 @@ class WriteCode(Action):
|
|||
|
||||
async def run(self, context, filename):
|
||||
prompt = PROMPT_TEMPLATE.format(context=context, filename=filename)
|
||||
logger.info(f'Writing {filename}..')
|
||||
logger.info(f"Writing {filename}..")
|
||||
code = await self.write_code(prompt)
|
||||
# code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING)
|
||||
# self._save(context, filename, code)
|
||||
|
|
|
|||
|
|
@ -7,9 +7,14 @@
|
|||
"""
|
||||
from typing import List, Tuple
|
||||
|
||||
import aiofiles
|
||||
|
||||
from metagpt.actions import Action, ActionOutput
|
||||
from metagpt.actions.search_and_summarize import SearchAndSummarize
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.common import CodeParser
|
||||
from metagpt.utils.mermaid import mermaid_to_file
|
||||
|
||||
PROMPT_TEMPLATE = """
|
||||
# Context
|
||||
|
|
@ -121,7 +126,7 @@ OUTPUT_MAPPING = {
|
|||
"Competitive Quadrant Chart": (str, ...),
|
||||
"Requirement Analysis": (str, ...),
|
||||
"Requirement Pool": (List[Tuple[str, str]], ...),
|
||||
"UI Design draft":(str, ...),
|
||||
"UI Design draft": (str, ...),
|
||||
"Anything UNCLEAR": (str, ...),
|
||||
}
|
||||
|
||||
|
|
@ -139,8 +144,31 @@ class WritePRD(Action):
|
|||
logger.info(sas.result)
|
||||
logger.info(rsp)
|
||||
|
||||
prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info,
|
||||
format_example=FORMAT_EXAMPLE)
|
||||
prompt = PROMPT_TEMPLATE.format(
|
||||
requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE
|
||||
)
|
||||
logger.debug(prompt)
|
||||
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
|
||||
|
||||
await self._save(prd.content)
|
||||
return prd
|
||||
|
||||
async def _save_prd(self, docs_path, resources_path, prd):
|
||||
prd_file = docs_path / "prd.md"
|
||||
quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd)
|
||||
await mermaid_to_file(
|
||||
mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis"
|
||||
)
|
||||
async with aiofiles.open(prd_file, "w") as f:
|
||||
await f.write(prd)
|
||||
logger.info(f"Saving PRD to {prd_file}")
|
||||
|
||||
async def _save(self, prd):
|
||||
workspace = CONFIG.workspace
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
docs_path = workspace / "docs"
|
||||
resources_path = workspace / "resources"
|
||||
docs_path.mkdir(parents=True, exist_ok=True)
|
||||
resources_path.mkdir(parents=True, exist_ok=True)
|
||||
await self._save_prd(docs_path, resources_path, prd)
|
||||
|
|
|
|||
|
|
@ -4,15 +4,17 @@
|
|||
Provide configuration, singleton.
|
||||
@Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`.
|
||||
"""
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
from copy import deepcopy
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
import openai
|
||||
import yaml
|
||||
|
||||
from metagpt.const import PROJECT_ROOT, OPTIONS
|
||||
from metagpt.const import OPTIONS, PROJECT_ROOT, WORKSPACE_ROOT
|
||||
from metagpt.logs import logger
|
||||
from metagpt.tools import SearchEngineType, WebBrowserEngineType
|
||||
from metagpt.utils.cost_manager import CostManager
|
||||
|
|
@ -55,7 +57,7 @@ class Config(metaclass=Singleton):
|
|||
self.openai_api_key = self._get("OPENAI_API_KEY")
|
||||
self.anthropic_api_key = self._get("Anthropic_API_KEY")
|
||||
if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and (
|
||||
not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key
|
||||
not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key
|
||||
):
|
||||
logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first")
|
||||
self.openai_api_base = self._get("OPENAI_API_BASE")
|
||||
|
|
@ -93,6 +95,11 @@ class Config(metaclass=Singleton):
|
|||
self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY")
|
||||
self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT")
|
||||
|
||||
workspace_uid = (
|
||||
self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}"
|
||||
)
|
||||
self.workspace = WORKSPACE_ROOT / workspace_uid
|
||||
|
||||
def _init_with_config_files_and_env(self, yaml_file):
|
||||
"""从config/key.yaml / config/config.yaml / env三处按优先级递减加载"""
|
||||
configs = dict(os.environ)
|
||||
|
|
|
|||
|
|
@ -14,9 +14,11 @@ def get_project_root():
|
|||
"""逐级向上寻找项目根目录"""
|
||||
current_path = Path.cwd()
|
||||
while True:
|
||||
if (current_path / '.git').exists() or \
|
||||
(current_path / '.project_root').exists() or \
|
||||
(current_path / '.gitignore').exists():
|
||||
if (
|
||||
(current_path / ".git").exists()
|
||||
or (current_path / ".project_root").exists()
|
||||
or (current_path / ".gitignore").exists()
|
||||
):
|
||||
return current_path
|
||||
parent_path = current_path.parent
|
||||
if parent_path == current_path:
|
||||
|
|
@ -25,15 +27,15 @@ def get_project_root():
|
|||
|
||||
|
||||
PROJECT_ROOT = get_project_root()
|
||||
DATA_PATH = PROJECT_ROOT / 'data'
|
||||
WORKSPACE_ROOT = PROJECT_ROOT / 'workspace'
|
||||
PROMPT_PATH = PROJECT_ROOT / 'metagpt/prompts'
|
||||
UT_PATH = PROJECT_ROOT / 'data/ut'
|
||||
DATA_PATH = PROJECT_ROOT / "data"
|
||||
WORKSPACE_ROOT = PROJECT_ROOT / "workspace"
|
||||
PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts"
|
||||
UT_PATH = PROJECT_ROOT / "data/ut"
|
||||
SWAGGER_PATH = UT_PATH / "files/api/"
|
||||
UT_PY_PATH = UT_PATH / "files/ut/"
|
||||
API_QUESTIONS_PATH = UT_PATH / "files/question/"
|
||||
YAPI_URL = "http://yapi.deepwisdomai.com/"
|
||||
TMP = PROJECT_ROOT / 'tmp'
|
||||
TMP = PROJECT_ROOT / "tmp"
|
||||
RESEARCH_PATH = DATA_PATH / "research"
|
||||
|
||||
MEM_TTL = 24 * 30 * 3600
|
||||
|
|
@ -43,4 +45,12 @@ DEFAULT_LANGUAGE = "English"
|
|||
DEFAULT_MAX_TOKENS = 1500
|
||||
COMMAND_TOKENS = 500
|
||||
BRAIN_MEMORY = "BRAIN_MEMORY"
|
||||
SKILL_PATH = "SKILL_PATH"
|
||||
SKILL_PATH = "SKILL_PATH"
|
||||
SERPER_API_KEY = "SERPER_API_KEY"
|
||||
|
||||
# Key Definitions for MetaGPT LLM
|
||||
METAGPT_API_MODEL = "METAGPT_API_MODEL"
|
||||
METAGPT_API_KEY = "METAGPT_API_KEY"
|
||||
METAGPT_API_BASE = "METAGPT_API_BASE"
|
||||
METAGPT_API_TYPE = "METAGPT_API_TYPE"
|
||||
METAGPT_API_VERSION = "METAGPT_API_VERSION"
|
||||
|
|
|
|||
|
|
@ -15,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot
|
|||
|
||||
class BaseGPTAPI(BaseChatbot):
|
||||
"""GPT API abstract class, requiring all inheritors to provide a series of standard capabilities"""
|
||||
system_prompt = 'You are a helpful assistant.'
|
||||
|
||||
system_prompt = "You are a helpful assistant."
|
||||
|
||||
def _user_msg(self, msg: str) -> dict[str, str]:
|
||||
return {"role": "user", "content": msg}
|
||||
|
|
@ -46,9 +47,9 @@ class BaseGPTAPI(BaseChatbot):
|
|||
rsp = await self.acompletion_text(message, stream=True)
|
||||
except Exception as e:
|
||||
logger.exception(f"{e}")
|
||||
logger.info(f"ask:{msg}, error:{e}")
|
||||
raise e
|
||||
logger.debug(message)
|
||||
# logger.debug(rsp)
|
||||
logger.info(f"ask:{msg}, anwser:{rsp}")
|
||||
return rsp
|
||||
|
||||
def _extract_assistant_rsp(self, context):
|
||||
|
|
@ -115,7 +116,7 @@ class BaseGPTAPI(BaseChatbot):
|
|||
|
||||
def messages_to_prompt(self, messages: list[dict]):
|
||||
"""[{"role": "user", "content": msg}] to user: <msg> etc."""
|
||||
return '\n'.join([f"{i['role']}: {i['content']}" for i in messages])
|
||||
return "\n".join([f"{i['role']}: {i['content']}" for i in messages])
|
||||
|
||||
def messages_to_dict(self, messages):
|
||||
"""objects to [{"role": "user", "content": msg}] etc."""
|
||||
|
|
|
|||
33
metagpt/provider/metagpt_llm_api.py
Normal file
33
metagpt/provider/metagpt_llm_api.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Time : 2023/8/30
|
||||
@Author : mashenquan
|
||||
@File : metagpt_llm_api.py
|
||||
@Desc : MetaGPT LLM related APIs
|
||||
"""
|
||||
|
||||
import openai
|
||||
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.provider import OpenAIGPTAPI
|
||||
from metagpt.provider.openai_api import RateLimiter
|
||||
|
||||
|
||||
class MetaGPTLLMAPI(OpenAIGPTAPI):
|
||||
"""MetaGPT LLM api"""
|
||||
|
||||
def __init__(self):
|
||||
self.__init_openai()
|
||||
self.llm = openai
|
||||
self.model = CONFIG.METAGPT_API_MODEL
|
||||
self.auto_max_tokens = False
|
||||
RateLimiter.__init__(self, rpm=self.rpm)
|
||||
|
||||
def __init_openai(self, *args, **kwargs):
|
||||
openai.api_key = CONFIG.METAGPT_API_KEY
|
||||
if CONFIG.METAGPT_API_BASE:
|
||||
openai.api_base = CONFIG.METAGPT_API_BASE
|
||||
if CONFIG.METAGPT_API_TYPE:
|
||||
openai.api_type = CONFIG.METAGPT_API_TYPE
|
||||
openai.api_version = CONFIG.METAGPT_API_VERSION
|
||||
self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10
|
||||
|
|
@ -6,17 +6,18 @@
|
|||
@File : engineer.py
|
||||
"""
|
||||
import asyncio
|
||||
import shutil
|
||||
from collections import OrderedDict
|
||||
from pathlib import Path
|
||||
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
import aiofiles
|
||||
|
||||
from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles import Role
|
||||
from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign
|
||||
from metagpt.schema import Message
|
||||
from metagpt.utils.common import CodeParser
|
||||
from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP
|
||||
from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP
|
||||
|
||||
|
||||
async def gather_ordered_k(coros, k) -> list:
|
||||
|
|
@ -47,9 +48,15 @@ async def gather_ordered_k(coros, k) -> list:
|
|||
|
||||
|
||||
class Engineer(Role):
|
||||
def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code",
|
||||
constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain",
|
||||
n_borg=1, use_code_review=False):
|
||||
def __init__(
|
||||
self,
|
||||
name="Alex",
|
||||
profile="Engineer",
|
||||
goal="Write elegant, readable, extensible, efficient code",
|
||||
constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain",
|
||||
n_borg=1,
|
||||
use_code_review=False,
|
||||
):
|
||||
super().__init__(name, profile, goal, constraints)
|
||||
self._init_actions([WriteCode])
|
||||
self.use_code_review = use_code_review
|
||||
|
|
@ -72,31 +79,24 @@ class Engineer(Role):
|
|||
@classmethod
|
||||
def parse_workspace(cls, system_design_msg: Message) -> str:
|
||||
if system_design_msg.instruct_content:
|
||||
return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip("\"")
|
||||
return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"')
|
||||
return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
|
||||
|
||||
def get_workspace(self) -> Path:
|
||||
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
|
||||
if not msg:
|
||||
return WORKSPACE_ROOT / 'src'
|
||||
return CONFIG.workspace / "src"
|
||||
workspace = self.parse_workspace(msg)
|
||||
# Codes are written in workspace/{package_name}/{package_name}
|
||||
return WORKSPACE_ROOT / workspace / workspace
|
||||
return CONFIG.workspace / workspace
|
||||
|
||||
def recreate_workspace(self):
|
||||
async def write_file(self, filename: str, code: str):
|
||||
workspace = self.get_workspace()
|
||||
try:
|
||||
shutil.rmtree(workspace)
|
||||
except FileNotFoundError:
|
||||
pass # 文件夹不存在,但我们不在意
|
||||
workspace.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def write_file(self, filename: str, code: str):
|
||||
workspace = self.get_workspace()
|
||||
filename = filename.replace('"', '').replace('\n', '')
|
||||
filename = filename.replace('"', "").replace("\n", "")
|
||||
file = workspace / filename
|
||||
file.parent.mkdir(parents=True, exist_ok=True)
|
||||
file.write_text(code)
|
||||
async with aiofiles.open(file, "w") as f:
|
||||
await f.write(code)
|
||||
return file
|
||||
|
||||
def recv(self, message: Message) -> None:
|
||||
|
|
@ -109,8 +109,7 @@ class Engineer(Role):
|
|||
todo_coros = []
|
||||
for todo in self.todos:
|
||||
todo_coro = WriteCode().run(
|
||||
context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]),
|
||||
filename=todo
|
||||
context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo
|
||||
)
|
||||
todo_coros.append(todo_coro)
|
||||
|
||||
|
|
@ -124,38 +123,40 @@ class Engineer(Role):
|
|||
self._rc.memory.add(msg)
|
||||
del self.todos[0]
|
||||
|
||||
logger.info(f'Done {self.get_workspace()} generating.')
|
||||
logger.info(f"Done {self.get_workspace()} generating.")
|
||||
msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo))
|
||||
return msg
|
||||
|
||||
async def _act_sp(self) -> Message:
|
||||
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
|
||||
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
|
||||
instruct_content = {}
|
||||
for todo in self.todos:
|
||||
code = await WriteCode().run(
|
||||
context=self._rc.history,
|
||||
filename=todo
|
||||
)
|
||||
code = await WriteCode().run(context=self._rc.history, filename=todo)
|
||||
# logger.info(todo)
|
||||
# logger.info(code_rsp)
|
||||
# code = self.parse_code(code_rsp)
|
||||
file_path = self.write_file(todo, code)
|
||||
file_path = await self.write_file(todo, code)
|
||||
msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo))
|
||||
self._rc.memory.add(msg)
|
||||
instruct_content[todo] = code
|
||||
|
||||
code_msg = todo + FILENAME_CODE_SEP + str(file_path)
|
||||
# code_msg = todo + FILENAME_CODE_SEP + str(file_path)
|
||||
code_msg = (todo, file_path)
|
||||
code_msg_all.append(code_msg)
|
||||
|
||||
logger.info(f'Done {self.get_workspace()} generating.')
|
||||
logger.info(f"Done {self.get_workspace()} generating.")
|
||||
msg = Message(
|
||||
content=MSG_SEP.join(code_msg_all),
|
||||
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
|
||||
instruct_content=instruct_content,
|
||||
role=self.profile,
|
||||
cause_by=type(self._rc.todo),
|
||||
send_to="QaEngineer"
|
||||
send_to="QaEngineer",
|
||||
)
|
||||
return msg
|
||||
|
||||
async def _act_sp_precision(self) -> Message:
|
||||
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
|
||||
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
|
||||
instruct_content = {}
|
||||
for todo in self.todos:
|
||||
"""
|
||||
# 从历史信息中挑选必须的信息,以减少prompt长度(人工经验总结)
|
||||
|
|
@ -170,35 +171,30 @@ class Engineer(Role):
|
|||
context.append(m.content)
|
||||
context_str = "\n".join(context)
|
||||
# 编写code
|
||||
code = await WriteCode().run(
|
||||
context=context_str,
|
||||
filename=todo
|
||||
)
|
||||
code = await WriteCode().run(context=context_str, filename=todo)
|
||||
# code review
|
||||
if self.use_code_review:
|
||||
try:
|
||||
rewrite_code = await WriteCodeReview().run(
|
||||
context=context_str,
|
||||
code=code,
|
||||
filename=todo
|
||||
)
|
||||
rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo)
|
||||
code = rewrite_code
|
||||
except Exception as e:
|
||||
logger.error("code review failed!", e)
|
||||
pass
|
||||
file_path = self.write_file(todo, code)
|
||||
file_path = await self.write_file(todo, code)
|
||||
msg = Message(content=code, role=self.profile, cause_by=WriteCode)
|
||||
self._rc.memory.add(msg)
|
||||
instruct_content[todo] = code
|
||||
|
||||
code_msg = todo + FILENAME_CODE_SEP + str(file_path)
|
||||
code_msg = (todo, file_path)
|
||||
code_msg_all.append(code_msg)
|
||||
|
||||
logger.info(f'Done {self.get_workspace()} generating.')
|
||||
logger.info(f"Done {self.get_workspace()} generating.")
|
||||
msg = Message(
|
||||
content=MSG_SEP.join(code_msg_all),
|
||||
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
|
||||
instruct_content=instruct_content,
|
||||
role=self.profile,
|
||||
cause_by=type(self._rc.todo),
|
||||
send_to="QaEngineer"
|
||||
send_to="QaEngineer",
|
||||
)
|
||||
return msg
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import os
|
|||
from pathlib import Path
|
||||
|
||||
from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles import Role
|
||||
from metagpt.schema import Message
|
||||
|
|
@ -43,13 +43,13 @@ class QaEngineer(Role):
|
|||
def get_workspace(self, return_proj_dir=True) -> Path:
|
||||
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
|
||||
if not msg:
|
||||
return WORKSPACE_ROOT / "src"
|
||||
return CONFIG.workspace / "src"
|
||||
workspace = self.parse_workspace(msg)
|
||||
# project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc.
|
||||
if return_proj_dir:
|
||||
return WORKSPACE_ROOT / workspace
|
||||
return CONFIG.workspace / workspace
|
||||
# development codes directory: workspace/{package_name}/{package_name}
|
||||
return WORKSPACE_ROOT / workspace / workspace
|
||||
return CONFIG.workspace / workspace / workspace
|
||||
|
||||
def write_file(self, filename: str, code: str):
|
||||
workspace = self.get_workspace() / "tests"
|
||||
|
|
|
|||
|
|
@ -11,15 +11,14 @@ from __future__ import annotations
|
|||
|
||||
from typing import Iterable, Type
|
||||
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from metagpt.actions import Action, ActionOutput
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.const import OPTIONS
|
||||
from metagpt.llm import LLM
|
||||
from metagpt.actions import Action, ActionOutput
|
||||
from metagpt.logs import logger
|
||||
from metagpt.memory import Memory, LongTermMemory
|
||||
from metagpt.memory import LongTermMemory, Memory
|
||||
from metagpt.schema import Message, MessageTag
|
||||
|
||||
PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """
|
||||
|
|
@ -52,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi
|
|||
|
||||
class RoleSetting(BaseModel):
|
||||
"""Role properties"""
|
||||
|
||||
name: str
|
||||
profile: str
|
||||
goal: str
|
||||
|
|
@ -67,7 +67,8 @@ class RoleSetting(BaseModel):
|
|||
|
||||
class RoleContext(BaseModel):
|
||||
"""Runtime role context"""
|
||||
env: 'Environment' = Field(default=None)
|
||||
|
||||
env: "Environment" = Field(default=None)
|
||||
memory: Memory = Field(default_factory=Memory)
|
||||
long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory)
|
||||
state: int = Field(default=0)
|
||||
|
|
@ -95,7 +96,7 @@ class RoleContext(BaseModel):
|
|||
@property
|
||||
def prerequisite(self):
|
||||
"""Retrieve information with `prerequisite` tag"""
|
||||
if self.memory and hasattr(self.memory, 'get_by_tags'):
|
||||
if self.memory and hasattr(self.memory, "get_by_tags"):
|
||||
return self.memory.get_by_tags([MessageTag.Prerequisite.value])
|
||||
return ""
|
||||
|
||||
|
|
@ -145,7 +146,7 @@ class Role:
|
|||
logger.debug(self._actions)
|
||||
self._rc.todo = self._actions[self._rc.state]
|
||||
|
||||
def set_env(self, env: 'Environment'):
|
||||
def set_env(self, env: "Environment"):
|
||||
"""设置角色工作所处的环境,角色可以向环境说话,也可以通过观察接受环境消息"""
|
||||
self._rc.env = env
|
||||
|
||||
|
|
@ -192,12 +193,13 @@ class Role:
|
|||
self._set_state(0)
|
||||
return True
|
||||
prompt = self._get_prefix()
|
||||
prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states),
|
||||
n_states=len(self._states) - 1)
|
||||
prompt += STATE_TEMPLATE.format(
|
||||
history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1
|
||||
)
|
||||
next_state = await self._llm.aask(prompt)
|
||||
logger.debug(f"{prompt=}")
|
||||
if not next_state.isdigit() or int(next_state) not in range(len(self._states)):
|
||||
logger.warning(f'Invalid answer of state, {next_state=}')
|
||||
logger.warning(f"Invalid answer of state, {next_state=}")
|
||||
next_state = "0"
|
||||
self._set_state(int(next_state))
|
||||
return True
|
||||
|
|
@ -212,8 +214,12 @@ class Role:
|
|||
response = await self._rc.todo.run(requirement)
|
||||
# logger.info(response)
|
||||
if isinstance(response, ActionOutput):
|
||||
msg = Message(content=response.content, instruct_content=response.instruct_content,
|
||||
role=self.profile, cause_by=type(self._rc.todo))
|
||||
msg = Message(
|
||||
content=response.content,
|
||||
instruct_content=response.instruct_content,
|
||||
role=self.profile,
|
||||
cause_by=type(self._rc.todo),
|
||||
)
|
||||
else:
|
||||
msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo))
|
||||
self._rc.memory.add(msg)
|
||||
|
|
@ -236,7 +242,7 @@ class Role:
|
|||
|
||||
news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news]
|
||||
if news_text:
|
||||
logger.debug(f'{self._setting} observed: {news_text}')
|
||||
logger.debug(f"{self._setting} observed: {news_text}")
|
||||
return len(self._rc.news)
|
||||
|
||||
def _publish_message(self, msg):
|
||||
|
|
@ -310,20 +316,15 @@ class Role:
|
|||
def add_to_do(self, act):
|
||||
self._rc.todo = act
|
||||
|
||||
async def think(self) -> bool:
|
||||
async def think(self) -> Action:
|
||||
"""The exported `think` function"""
|
||||
has_action = await self._think()
|
||||
if not has_action:
|
||||
return False
|
||||
if not self._rc.todo:
|
||||
return False
|
||||
return True
|
||||
await self._think()
|
||||
return self._rc.todo
|
||||
|
||||
async def act(self) -> ActionOutput:
|
||||
"""The exported `act` function"""
|
||||
msg = await self._act()
|
||||
return ActionOutput(content=msg.content,
|
||||
instruct_content=msg.instruct_content)
|
||||
return ActionOutput(content=msg.content, instruct_content=msg.instruct_content)
|
||||
|
||||
@property
|
||||
def todo_description(self):
|
||||
|
|
|
|||
|
|
@ -9,22 +9,34 @@
|
|||
"""
|
||||
|
||||
|
||||
import re
|
||||
|
||||
import aiofiles
|
||||
|
||||
from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.actions.write_teaching_plan import (
|
||||
TeachingPlanRequirement,
|
||||
WriteTeachingPlanPart,
|
||||
)
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles import Role
|
||||
from metagpt.schema import Message
|
||||
from metagpt.logs import logger
|
||||
import re
|
||||
|
||||
|
||||
class Teacher(Role):
|
||||
"""Support configurable teacher roles,
|
||||
with native and teaching languages being replaceable through configurations."""
|
||||
def __init__(self, name='Lily', profile='{teaching_language} Teacher',
|
||||
goal='writing a {language} teaching plan part by part',
|
||||
constraints='writing in {language}', desc="", *args, **kwargs):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name="Lily",
|
||||
profile="{teaching_language} Teacher",
|
||||
goal="writing a {language} teaching plan part by part",
|
||||
constraints="writing in {language}",
|
||||
desc="",
|
||||
*args,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs)
|
||||
actions = []
|
||||
for topic in WriteTeachingPlanPart.TOPICS:
|
||||
|
|
@ -54,7 +66,7 @@ class Teacher(Role):
|
|||
break
|
||||
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
|
||||
msg = await self._act()
|
||||
if ret.content != '':
|
||||
if ret.content != "":
|
||||
ret.content += "\n\n\n"
|
||||
ret.content += msg.content
|
||||
logger.info(ret.content)
|
||||
|
|
@ -64,14 +76,14 @@ class Teacher(Role):
|
|||
async def save(self, content):
|
||||
"""Save teaching plan"""
|
||||
filename = Teacher.new_file_name(self.course_title)
|
||||
pathname = WORKSPACE_ROOT / "teaching_plan"
|
||||
pathname = CONFIG.workspace / "teaching_plan"
|
||||
pathname.mkdir(exist_ok=True)
|
||||
pathname = pathname / filename
|
||||
try:
|
||||
async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer:
|
||||
async with aiofiles.open(str(pathname), mode="w", encoding="utf-8") as writer:
|
||||
await writer.write(content)
|
||||
except Exception as e:
|
||||
logger.error(f'Save failed:{e}')
|
||||
logger.error(f"Save failed:{e}")
|
||||
logger.info(f"Save to:{pathname}")
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -80,8 +92,8 @@ class Teacher(Role):
|
|||
# Define the special characters that need to be replaced.
|
||||
illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']'
|
||||
# Replace the special characters with underscores.
|
||||
filename = re.sub(illegal_chars, '_', lesson_title) + ext
|
||||
return re.sub(r'_+', '_', filename)
|
||||
filename = re.sub(illegal_chars, "_", lesson_title) + ext
|
||||
return re.sub(r"_+", "_", filename)
|
||||
|
||||
@property
|
||||
def course_title(self):
|
||||
|
|
@ -93,9 +105,9 @@ class Teacher(Role):
|
|||
if act.rsp is None:
|
||||
return default_title
|
||||
title = act.rsp.lstrip("# \n")
|
||||
if '\n' in title:
|
||||
ix = title.index('\n')
|
||||
title = title[0: ix]
|
||||
if "\n" in title:
|
||||
ix = title.index("\n")
|
||||
title = title[0:ix]
|
||||
return title
|
||||
|
||||
return default_title
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@ from aiohttp import ClientSession
|
|||
from PIL import Image, PngImagePlugin
|
||||
|
||||
from metagpt.config import Config
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.logs import logger
|
||||
|
||||
config = Config()
|
||||
|
|
@ -81,7 +80,7 @@ class SDEngine:
|
|||
return self.payload
|
||||
|
||||
def _save(self, imgs, save_name=""):
|
||||
save_dir = WORKSPACE_ROOT / "resources" / "SD_Output"
|
||||
save_dir = CONFIG.get_workspace() / "resources" / "SD_Output"
|
||||
if not os.path.exists(save_dir):
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
batch_decode_base64_to_image(imgs, save_dir, save_name=save_name)
|
||||
|
|
|
|||
|
|
@ -8,11 +8,11 @@ from __future__ import annotations
|
|||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Literal, Dict
|
||||
from typing import Literal
|
||||
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from metagpt.config import Config
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.parse_html import WebPage
|
||||
|
||||
|
|
@ -28,20 +28,18 @@ class PlaywrightWrapper:
|
|||
|
||||
def __init__(
|
||||
self,
|
||||
options: Dict,
|
||||
browser_type: Literal["chromium", "firefox", "webkit"] | None = None,
|
||||
launch_kwargs: dict | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
self.options = options
|
||||
if browser_type is None:
|
||||
browser_type = options.get("playwright_browser_type")
|
||||
browser_type = CONFIG.playwright_browser_type
|
||||
self.browser_type = browser_type
|
||||
launch_kwargs = launch_kwargs or {}
|
||||
if options.get("global_proxy") and "proxy" not in launch_kwargs:
|
||||
if CONFIG.global_proxy and "proxy" not in launch_kwargs:
|
||||
args = launch_kwargs.get("args", [])
|
||||
if not any(str.startswith(i, "--proxy-server=") for i in args):
|
||||
launch_kwargs["proxy"] = {"server": options.get("global_proxy")}
|
||||
launch_kwargs["proxy"] = {"server": CONFIG.global_proxy}
|
||||
self.launch_kwargs = launch_kwargs
|
||||
context_kwargs = {}
|
||||
if "ignore_https_errors" in kwargs:
|
||||
|
|
@ -81,8 +79,8 @@ class PlaywrightWrapper:
|
|||
executable_path = Path(browser_type.executable_path)
|
||||
if not executable_path.exists() and "executable_path" not in self.launch_kwargs:
|
||||
kwargs = {}
|
||||
if self.options.get("global_proxy"):
|
||||
kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")}
|
||||
if CONFIG.global_proxy:
|
||||
kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy}
|
||||
await _install_browsers(self.browser_type, **kwargs)
|
||||
|
||||
if self._has_run_precheck:
|
||||
|
|
@ -150,8 +148,6 @@ if __name__ == "__main__":
|
|||
import fire
|
||||
|
||||
async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs):
|
||||
return await PlaywrightWrapper(options=Config().runtime_options,
|
||||
browser_type=browser_type,
|
||||
**kwargs).run(url, *urls)
|
||||
return await PlaywrightWrapper(browser_type=browser_type, **kwargs).run(url, *urls)
|
||||
|
||||
fire.Fire(main)
|
||||
|
|
|
|||
|
|
@ -6,19 +6,20 @@
|
|||
@File : mermaid.py
|
||||
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
|
||||
"""
|
||||
import subprocess
|
||||
import asyncio
|
||||
from pathlib import Path
|
||||
|
||||
from metagpt.config import Config
|
||||
# from metagpt.utils.common import check_cmd_exists
|
||||
import aiofiles
|
||||
|
||||
from metagpt.config import CONFIG, Config
|
||||
from metagpt.const import PROJECT_ROOT
|
||||
from metagpt.logs import logger
|
||||
from metagpt.utils.common import check_cmd_exists
|
||||
|
||||
|
||||
def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
|
||||
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
|
||||
"""suffix: png/svg/pdf
|
||||
|
||||
:param options: runtime context options, created by `Config` class object and changed in flow pipeline
|
||||
:param mermaid_code: mermaid code
|
||||
:param output_file_without_suffix: output filename
|
||||
:param width:
|
||||
|
|
@ -27,92 +28,87 @@ def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=204
|
|||
"""
|
||||
# Write the Mermaid code to a temporary file
|
||||
tmp = Path(f"{output_file_without_suffix}.mmd")
|
||||
tmp.write_text(mermaid_code, encoding="utf-8")
|
||||
async with aiofiles.open(tmp, "w", encoding="utf-8") as f:
|
||||
await f.write(mermaid_code)
|
||||
# tmp.write_text(mermaid_code, encoding="utf-8")
|
||||
|
||||
if check_cmd_exists("mmdc") != 0:
|
||||
logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
|
||||
return -1
|
||||
# if check_cmd_exists("mmdc") != 0:
|
||||
# logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
|
||||
# return -1
|
||||
|
||||
for suffix in ["pdf", "svg", "png"]:
|
||||
# for suffix in ["pdf", "svg", "png"]:
|
||||
for suffix in ["png"]:
|
||||
output_file = f"{output_file_without_suffix}.{suffix}"
|
||||
# Call the `mmdc` command to convert the Mermaid code to a PNG
|
||||
logger.info(f"Generating {output_file}..")
|
||||
cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
|
||||
|
||||
if options.get("puppeteer_config"):
|
||||
subprocess.run(
|
||||
[
|
||||
options.get("mmdc"),
|
||||
"-p",
|
||||
options.get("puppeteer_config"),
|
||||
"-i",
|
||||
str(tmp),
|
||||
"-o",
|
||||
output_file,
|
||||
"-w",
|
||||
str(width),
|
||||
"-H",
|
||||
str(height),
|
||||
]
|
||||
)
|
||||
else:
|
||||
subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)])
|
||||
return 0
|
||||
|
||||
|
||||
MMC1 = """classDiagram
|
||||
class Main {
|
||||
-SearchEngine search_engine
|
||||
+main() str
|
||||
}
|
||||
class SearchEngine {
|
||||
-Index index
|
||||
-Ranking ranking
|
||||
-Summary summary
|
||||
+search(query: str) str
|
||||
}
|
||||
class Index {
|
||||
-KnowledgeBase knowledge_base
|
||||
+create_index(data: dict)
|
||||
+query_index(query: str) list
|
||||
}
|
||||
class Ranking {
|
||||
+rank_results(results: list) list
|
||||
}
|
||||
class Summary {
|
||||
+summarize_results(results: list) str
|
||||
}
|
||||
class KnowledgeBase {
|
||||
+update(data: dict)
|
||||
+fetch_data(query: str) dict
|
||||
}
|
||||
Main --> SearchEngine
|
||||
SearchEngine --> Index
|
||||
SearchEngine --> Ranking
|
||||
SearchEngine --> Summary
|
||||
Index --> KnowledgeBase"""
|
||||
|
||||
MMC2 = """sequenceDiagram
|
||||
participant M as Main
|
||||
participant SE as SearchEngine
|
||||
participant I as Index
|
||||
participant R as Ranking
|
||||
participant S as Summary
|
||||
participant KB as KnowledgeBase
|
||||
M->>SE: search(query)
|
||||
SE->>I: query_index(query)
|
||||
I->>KB: fetch_data(query)
|
||||
KB-->>I: return data
|
||||
I-->>SE: return results
|
||||
SE->>R: rank_results(results)
|
||||
R-->>SE: return ranked_results
|
||||
SE->>S: summarize_results(ranked_results)
|
||||
S-->>SE: return summary
|
||||
SE-->>M: return summary"""
|
||||
if CONFIG.puppeteer_config:
|
||||
cmds.extend(["-p", CONFIG.puppeteer_config])
|
||||
process = await asyncio.create_subprocess_exec(*cmds)
|
||||
await process.wait()
|
||||
return process.returncode
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
MMC1 = """classDiagram
|
||||
class Main {
|
||||
-SearchEngine search_engine
|
||||
+main() str
|
||||
}
|
||||
class SearchEngine {
|
||||
-Index index
|
||||
-Ranking ranking
|
||||
-Summary summary
|
||||
+search(query: str) str
|
||||
}
|
||||
class Index {
|
||||
-KnowledgeBase knowledge_base
|
||||
+create_index(data: dict)
|
||||
+query_index(query: str) list
|
||||
}
|
||||
class Ranking {
|
||||
+rank_results(results: list) list
|
||||
}
|
||||
class Summary {
|
||||
+summarize_results(results: list) str
|
||||
}
|
||||
class KnowledgeBase {
|
||||
+update(data: dict)
|
||||
+fetch_data(query: str) dict
|
||||
}
|
||||
Main --> SearchEngine
|
||||
SearchEngine --> Index
|
||||
SearchEngine --> Ranking
|
||||
SearchEngine --> Summary
|
||||
Index --> KnowledgeBase"""
|
||||
|
||||
MMC2 = """sequenceDiagram
|
||||
participant M as Main
|
||||
participant SE as SearchEngine
|
||||
participant I as Index
|
||||
participant R as Ranking
|
||||
participant S as Summary
|
||||
participant KB as KnowledgeBase
|
||||
M->>SE: search(query)
|
||||
SE->>I: query_index(query)
|
||||
I->>KB: fetch_data(query)
|
||||
KB-->>I: return data
|
||||
I-->>SE: return results
|
||||
SE->>R: rank_results(results)
|
||||
R-->>SE: return ranked_results
|
||||
SE->>S: summarize_results(ranked_results)
|
||||
S-->>SE: return summary
|
||||
SE-->>M: return summary"""
|
||||
|
||||
conf = Config()
|
||||
mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1,
|
||||
output_file_without_suffix=PROJECT_ROOT / "tmp/1.png")
|
||||
mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2,
|
||||
output_file_without_suffix=PROJECT_ROOT / "tmp/2.png")
|
||||
asyncio.run(
|
||||
mermaid_to_file(
|
||||
options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png"
|
||||
)
|
||||
)
|
||||
asyncio.run(
|
||||
mermaid_to_file(
|
||||
options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png"
|
||||
)
|
||||
)
|
||||
|
|
|
|||
127
metagpt/utils/s3.py
Normal file
127
metagpt/utils/s3.py
Normal file
|
|
@ -0,0 +1,127 @@
|
|||
|
||||
from typing import Optional
|
||||
|
||||
import aioboto3
|
||||
from metagpt.logs import logger
|
||||
from metagpt.config import Config
|
||||
|
||||
|
||||
class S3:
|
||||
"""A class for interacting with Amazon S3 storage."""
|
||||
|
||||
def __init__(self):
|
||||
self.session = aioboto3.Session()
|
||||
self.s3_config = Config().get("S3")
|
||||
self.auth_config = {
|
||||
"service_name": "s3",
|
||||
"aws_access_key_id": self.s3_config["access_key"],
|
||||
"aws_secret_access_key": self.s3_config["secret_key"],
|
||||
"endpoint_url": self.s3_config["endpoint_url"]
|
||||
}
|
||||
|
||||
async def upload_file(
|
||||
self,
|
||||
bucket: str,
|
||||
local_path: str,
|
||||
object_name: str,
|
||||
) -> None:
|
||||
"""Upload a file from the local path to the specified path of the storage bucket specified in s3.
|
||||
|
||||
Args:
|
||||
bucket: The name of the S3 storage bucket.
|
||||
local_path: The local file path, including the file name.
|
||||
object_name: The complete path of the uploaded file to be stored in S3, including the file name.
|
||||
|
||||
Raises:
|
||||
Exception: If an error occurs during the upload process, an exception is raised.
|
||||
"""
|
||||
try:
|
||||
async with self.session.client(**self.auth_config) as client:
|
||||
with open(local_path, "rb") as file:
|
||||
await client.put_object(Body=file, Bucket=bucket, Key=object_name)
|
||||
logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.")
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}")
|
||||
raise e
|
||||
|
||||
async def get_object_url(
|
||||
self,
|
||||
bucket: str,
|
||||
object_name: str,
|
||||
) -> str:
|
||||
"""Get the URL for a downloadable or preview file stored in the specified S3 bucket.
|
||||
|
||||
Args:
|
||||
bucket: The name of the S3 storage bucket.
|
||||
object_name: The complete path of the file stored in S3, including the file name.
|
||||
|
||||
Returns:
|
||||
The URL for the downloadable or preview file.
|
||||
|
||||
Raises:
|
||||
Exception: If an error occurs while retrieving the URL, an exception is raised.
|
||||
"""
|
||||
try:
|
||||
async with self.session.client(**self.auth_config) as client:
|
||||
file = await client.get_object(Bucket=bucket, Key=object_name)
|
||||
return str(file["Body"].url)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get the url for a downloadable or preview file: {e}")
|
||||
raise e
|
||||
|
||||
async def get_object(
|
||||
self,
|
||||
bucket: str,
|
||||
object_name: str,
|
||||
) -> bytes:
|
||||
"""Get the binary data of a file stored in the specified S3 bucket.
|
||||
|
||||
Args:
|
||||
bucket: The name of the S3 storage bucket.
|
||||
object_name: The complete path of the file stored in S3, including the file name.
|
||||
|
||||
Returns:
|
||||
The binary data of the requested file.
|
||||
|
||||
Raises:
|
||||
Exception: If an error occurs while retrieving the file data, an exception is raised.
|
||||
"""
|
||||
try:
|
||||
async with self.session.client(**self.auth_config) as client:
|
||||
s3_object = await client.get_object(Bucket=bucket, Key=object_name)
|
||||
return await s3_object["Body"].read()
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to get the binary data of the file: {e}")
|
||||
raise e
|
||||
|
||||
async def download_file(
|
||||
self,
|
||||
bucket: str,
|
||||
object_name: str,
|
||||
local_path: str,
|
||||
chunk_size: Optional[int] = 128 * 1024
|
||||
) -> None:
|
||||
"""Download an S3 object to a local file.
|
||||
|
||||
Args:
|
||||
bucket: The name of the S3 storage bucket.
|
||||
object_name: The complete path of the file stored in S3, including the file name.
|
||||
local_path: The local file path where the S3 object will be downloaded.
|
||||
chunk_size: The size of data chunks to read and write at a time. Default is 128 KB.
|
||||
|
||||
Raises:
|
||||
Exception: If an error occurs during the download process, an exception is raised.
|
||||
"""
|
||||
try:
|
||||
async with self.session.client(**self.auth_config) as client:
|
||||
s3_object = await client.get_object(Bucket=bucket, Key=object_name)
|
||||
stream = s3_object["Body"]
|
||||
with open(local_path, 'wb') as local_file:
|
||||
while True:
|
||||
file_data = await stream.read(chunk_size)
|
||||
if not file_data:
|
||||
break
|
||||
local_file.write(file_data)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to download the file from S3: {e}")
|
||||
raise e
|
||||
|
|
@ -40,4 +40,5 @@ libcst==1.0.1
|
|||
qdrant-client==1.4.0
|
||||
connexion[swagger-ui]
|
||||
aiohttp_jinja2
|
||||
azure-cognitiveservices-speech==1.31.0
|
||||
azure-cognitiveservices-speech==1.31.0
|
||||
aioboto3~=11.3.0
|
||||
|
|
@ -9,7 +9,6 @@
|
|||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
|
||||
from metagpt.config import Config
|
||||
from metagpt.logs import logger
|
||||
|
|
@ -74,3 +73,4 @@ def proxy():
|
|||
@pytest.fixture(scope="session", autouse=True)
|
||||
def init_config():
|
||||
Config()
|
||||
|
||||
|
|
|
|||
17
tests/metagpt/provider/test_metagpt_llm_api.py
Normal file
17
tests/metagpt/provider/test_metagpt_llm_api.py
Normal file
|
|
@ -0,0 +1,17 @@
|
|||
#!/usr/bin/env python
|
||||
# -*- coding: utf-8 -*-
|
||||
"""
|
||||
@Time : 2023/8/30
|
||||
@Author : mashenquan
|
||||
@File : test_metagpt_llm_api.py
|
||||
"""
|
||||
from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI
|
||||
|
||||
|
||||
def test_metagpt():
|
||||
llm = MetaGPTLLMAPI()
|
||||
assert llm
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_metagpt()
|
||||
|
|
@ -8,7 +8,7 @@ from functools import wraps
|
|||
from importlib import import_module
|
||||
|
||||
from metagpt.actions import Action, ActionOutput, WritePRD
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.logs import logger
|
||||
from metagpt.roles import Role
|
||||
from metagpt.schema import Message
|
||||
|
|
@ -214,7 +214,7 @@ class UIDesign(Action):
|
|||
logger.info("Finish icon design using StableDiffusion API")
|
||||
|
||||
async def _save(self, css_content, html_content):
|
||||
save_dir = WORKSPACE_ROOT / "resources" / "codes"
|
||||
save_dir = CONFIG.workspace / "resources" / "codes"
|
||||
if not os.path.exists(save_dir):
|
||||
os.makedirs(save_dir, exist_ok=True)
|
||||
# Save CSS and HTML content to files
|
||||
|
|
|
|||
|
|
@ -8,11 +8,8 @@
|
|||
@Modified By: mashenquan, 2023-8-17, move to `tools` folder.
|
||||
"""
|
||||
import asyncio
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt'
|
||||
from metagpt.const import WORKSPACE_ROOT
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.tools.azure_tts import AzureTTS
|
||||
|
||||
|
||||
|
|
@ -28,15 +25,13 @@ def test_azure_tts():
|
|||
“Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.”
|
||||
</mstts:express-as>
|
||||
"""
|
||||
path = WORKSPACE_ROOT / "tts"
|
||||
path = CONFIG.workspace / "tts"
|
||||
path.mkdir(exist_ok=True, parents=True)
|
||||
filename = path / "girl.wav"
|
||||
loop = asyncio.new_event_loop()
|
||||
v = loop.create_task(azure_tts.synthesize_speech(
|
||||
lang="zh-CN",
|
||||
voice="zh-CN-XiaomoNeural",
|
||||
text=text,
|
||||
output_file=str(filename)))
|
||||
v = loop.create_task(
|
||||
azure_tts.synthesize_speech(lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, output_file=str(filename))
|
||||
)
|
||||
result = loop.run_until_complete(v)
|
||||
|
||||
print(result)
|
||||
|
|
@ -45,5 +40,5 @@ def test_azure_tts():
|
|||
# TODO: 这里如果要检验,还要额外加上对应的asr,才能确保前后生成是接近一致的,但现在还没有
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
if __name__ == "__main__":
|
||||
test_azure_tts()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@
|
|||
#
|
||||
import os
|
||||
|
||||
from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT
|
||||
from metagpt.config import CONFIG
|
||||
from metagpt.tools.sd_engine import SDEngine
|
||||
|
||||
|
||||
def test_sd_engine_init():
|
||||
|
|
@ -21,5 +22,5 @@ def test_sd_engine_generate_prompt():
|
|||
async def test_sd_engine_run_t2i():
|
||||
sd_engine = SDEngine()
|
||||
await sd_engine.run_t2i(prompts=["test"])
|
||||
img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png"
|
||||
img_path = CONFIG.workspace / "resources" / "SD_Output" / "output_0.png"
|
||||
assert os.path.exists(img_path) == True
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue