From dc14770e3d5ad327ec90e61c52346b9549d567fb Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 30 Aug 2023 10:53:47 +0800 Subject: [PATCH 01/11] separate workspace --- metagpt/actions/action.py | 12 +- metagpt/actions/design_api.py | 62 ++++------ metagpt/actions/project_management.py | 25 ++-- metagpt/actions/write_code.py | 24 +--- metagpt/actions/write_prd.py | 34 +++++- metagpt/config.py | 11 +- metagpt/roles/engineer.py | 94 +++++++-------- metagpt/roles/qa_engineer.py | 8 +- metagpt/roles/role.py | 43 +++---- metagpt/roles/teacher.py | 44 ++++--- metagpt/tools/sd_engine.py | 3 +- metagpt/utils/mermaid.py | 164 +++++++++++++------------- tests/metagpt/roles/ui_role.py | 4 +- tests/metagpt/tools/test_azure_tts.py | 17 +-- tests/metagpt/tools/test_sd_tool.py | 5 +- 15 files changed, 275 insertions(+), 275 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5cf4f3d81..e4b9613ad 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,6 +6,8 @@ @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. """ +from __future__ import annotations + from abc import ABC from typing import Optional @@ -13,12 +15,12 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM -from metagpt.utils.common import OutputParser from metagpt.logs import logger +from metagpt.utils.common import OutputParser class Action(ABC): - def __init__(self, name: str = '', context=None, llm: LLM = None): + def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name if llm is None: llm = LLM() @@ -49,9 +51,9 @@ class Action(ABC): return await self.llm.aask(prompt, system_msgs) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def _aask_v1(self, prompt: str, output_class_name: str, - output_data_mapping: dict, - system_msgs: Optional[list[str]] = None) -> ActionOutput: + async def _aask_v1( + self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None + ) -> ActionOutput: """Append default prefix""" if not system_msgs: system_msgs = [] diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index cf23e6ad1..1c31b75fb 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -6,12 +6,12 @@ @File : design_api.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ -import shutil -from pathlib import Path from typing import List -from metagpt.actions import Action, ActionOutput -from metagpt.const import WORKSPACE_ROOT +import aiofiles + +from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.mermaid import mermaid_to_file @@ -93,52 +93,32 @@ OUTPUT_MAPPING = { class WriteDesign(Action): def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) - self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ - "data structures, library tables, processes, and paths. Please provide your design, feedback " \ - "clearly and in detail." + self.desc = ( + "Based on the PRD, think about the system design, and design the corresponding APIs, " + "data structures, library tables, processes, and paths. Please provide your design, feedback " + "clearly and in detail." + ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # 文件夹不存在，但我们不在意 - workspace.mkdir(parents=True, exist_ok=True) - - def _save_prd(self, docs_path, resources_path, prd): - prd_file = docs_path / 'prd.md' - quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(prd) - - def _save_system_design(self, docs_path, resources_path, content): + async def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(data_api_design, resources_path / 'data_api_design') - mermaid_to_file(seq_flow, resources_path / 'seq_flow') - system_design_file = docs_path / 'system_design.md' + await mermaid_to_file(data_api_design, resources_path / "data_api_design") + await mermaid_to_file(seq_flow, resources_path / "seq_flow") + system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text(content) + async with aiofiles.open(system_design_file, "w") as f: + await f.write(content) - def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - content = system_design.content - ws_name = CodeParser.parse_str(block="Python package name", text=content) - else: - content = system_design - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / 'docs' - resources_path = workspace / 'resources' + async def _save(self, system_design: str): + workspace = CONFIG.workspace + docs_path = workspace / "docs" + resources_path = workspace / "resources" docs_path.mkdir(parents=True, exist_ok=True) resources_path.mkdir(parents=True, exist_ok=True) - self._save_prd(docs_path, resources_path, context[-1].content) - self._save_system_design(docs_path, resources_path, content) + await self._save_system_design(docs_path, resources_path, system_design) async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) - # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) - self._save(context, system_design) + await self._save(system_design.content) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 16473ff01..55e7cbcb5 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -8,11 +8,12 @@ """ from typing import List, Tuple -from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT -from metagpt.utils.common import CodeParser +import aiofiles -PROMPT_TEMPLATE = ''' +from metagpt.actions.action import Action +from metagpt.config import CONFIG + +PROMPT_TEMPLATE = """ # Context {context} @@ -37,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -''' +""" FORMAT_EXAMPLE = ''' --- @@ -103,23 +104,23 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - def _save(self, context, rsp): - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / 'docs/api_spec_and_tasks.md' + async def _save(self, rsp): + file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" file_path.write_text(rsp.content) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' - requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) + requirements_path = CONFIG.workspace / "requirements.txt" + + async with aiofiles.open(requirements_path, "w") as f: + await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) - self._save(context, rsp) + await self._save(rsp) return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index cc122ef7a..fd54ce699 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,12 @@ @Author : alexanderwu @File : write_code.py """ -from metagpt.actions import WriteDesign +from tenacity import retry, stop_after_attempt, wait_fixed + from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -49,23 +48,6 @@ class WriteCode(Action): def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == WriteDesign][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): code_rsp = await self._aask(prompt) @@ -74,7 +56,7 @@ class WriteCode(Action): async def run(self, context, filename): prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f'Writing {filename}..') + logger.info(f"Writing {filename}..") code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0edd24d55..97f9138fd 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -7,9 +7,14 @@ """ from typing import List, Tuple +import aiofiles + from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.utils.common import CodeParser +from metagpt.utils.mermaid import mermaid_to_file PROMPT_TEMPLATE = """ # Context @@ -121,7 +126,7 @@ OUTPUT_MAPPING = { "Competitive Quadrant Chart": (str, ...), "Requirement Analysis": (str, ...), "Requirement Pool": (List[Tuple[str, str]], ...), - "UI Design draft":(str, ...), + "UI Design draft": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -139,8 +144,31 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) - prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info, - format_example=FORMAT_EXAMPLE) + prompt = PROMPT_TEMPLATE.format( + requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE + ) logger.debug(prompt) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) + + await self._save(prd.content) return prd + + async def _save_prd(self, docs_path, resources_path, prd): + prd_file = docs_path / "prd.md" + quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) + await mermaid_to_file( + mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis" + ) + async with aiofiles.open(prd_file, "w") as f: + await f.write(prd) + logger.info(f"Saving PRD to {prd_file}") + + async def _save(self, prd): + workspace = CONFIG.workspace + workspace.mkdir(parents=True, exist_ok=True) + + docs_path = workspace / "docs" + resources_path = workspace / "resources" + docs_path.mkdir(parents=True, exist_ok=True) + resources_path.mkdir(parents=True, exist_ok=True) + await self._save_prd(docs_path, resources_path, prd) diff --git a/metagpt/config.py b/metagpt/config.py index 5944fef57..908faaaaf 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -4,15 +4,17 @@ Provide configuration, singleton. @Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ +import datetime import json import os from copy import deepcopy from typing import Any +from uuid import uuid4 import openai import yaml -from metagpt.const import PROJECT_ROOT, OPTIONS +from metagpt.const import OPTIONS, PROJECT_ROOT, WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.cost_manager import CostManager @@ -55,7 +57,7 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -93,6 +95,11 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") + workspace_uid = ( + self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}" + ) + self.workspace = WORKSPACE_ROOT / workspace_uid + def _init_with_config_files_and_env(self, yaml_file): """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" configs = dict(os.environ) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 072e53998..97d0af087 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -6,17 +6,18 @@ @File : engineer.py """ import asyncio -import shutil from collections import OrderedDict from pathlib import Path -from metagpt.const import WORKSPACE_ROOT +import aiofiles + +from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role -from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign from metagpt.schema import Message from metagpt.utils.common import CodeParser -from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP +from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP async def gather_ordered_k(coros, k) -> list: @@ -47,9 +48,15 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", - n_borg=1, use_code_review=False): + def __init__( + self, + name="Alex", + profile="Engineer", + goal="Write elegant, readable, extensible, efficient code", + constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", + n_borg=1, + use_code_review=False, + ): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review @@ -72,31 +79,24 @@ class Engineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip("\"") + return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / 'src' + return CONFIG.workspace / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace / workspace - def recreate_workspace(self): + async def write_file(self, filename: str, code: str): workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # 文件夹不存在，但我们不在意 - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', '').replace('\n', '') + filename = filename.replace('"', "").replace("\n", "") file = workspace / filename file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) + async with aiofiles.open(file, "w") as f: + await f.write(code) return file def recv(self, message: Message) -> None: @@ -109,8 +109,7 @@ class Engineer(Role): todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), - filename=todo + context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo ) todo_coros.append(todo_coro) @@ -124,38 +123,40 @@ class Engineer(Role): self._rc.memory.add(msg) del self.todos[0] - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + instruct_content = {} for todo in self.todos: - code = await WriteCode().run( - context=self._rc.history, - filename=todo - ) + code = await WriteCode().run(context=self._rc.history, filename=todo) # logger.info(todo) # logger.info(code_rsp) # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) + file_path = await self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) + instruct_content[todo] = code - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + # code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = (todo, file_path) code_msg_all.append(code_msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), + instruct_content=instruct_content, role=self.profile, cause_by=type(self._rc.todo), - send_to="QaEngineer" + send_to="QaEngineer", ) return msg async def _act_sp_precision(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + instruct_content = {} for todo in self.todos: """ # 从历史信息中挑选必须的信息，以减少prompt长度（人工经验总结） @@ -170,35 +171,30 @@ class Engineer(Role): context.append(m.content) context_str = "\n".join(context) # 编写code - code = await WriteCode().run( - context=context_str, - filename=todo - ) + code = await WriteCode().run(context=context_str, filename=todo) # code review if self.use_code_review: try: - rewrite_code = await WriteCodeReview().run( - context=context_str, - code=code, - filename=todo - ) + rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) code = rewrite_code except Exception as e: logger.error("code review failed!", e) pass - file_path = self.write_file(todo, code) + file_path = await self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) + instruct_content[todo] = code - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = (todo, file_path) code_msg_all.append(code_msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), + instruct_content=instruct_content, role=self.profile, cause_by=type(self._rc.todo), - send_to="QaEngineer" + send_to="QaEngineer", ) return msg diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 65bf2cc5b..491f5f997 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -9,7 +9,7 @@ import os from pathlib import Path from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -43,13 +43,13 @@ class QaEngineer(Role): def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index aba7d4574..2f0f713f8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -11,15 +11,14 @@ from __future__ import annotations from typing import Iterable, Type - from pydantic import BaseModel, Field +from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import OPTIONS from metagpt.llm import LLM -from metagpt.actions import Action, ActionOutput from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageTag PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -52,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi class RoleSetting(BaseModel): """Role properties""" + name: str profile: str goal: str @@ -67,7 +67,8 @@ class RoleSetting(BaseModel): class RoleContext(BaseModel): """Runtime role context""" - env: 'Environment' = Field(default=None) + + env: "Environment" = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=0) @@ -95,7 +96,7 @@ class RoleContext(BaseModel): @property def prerequisite(self): """Retrieve information with `prerequisite` tag""" - if self.memory and hasattr(self.memory, 'get_by_tags'): + if self.memory and hasattr(self.memory, "get_by_tags"): return self.memory.get_by_tags([MessageTag.Prerequisite.value]) return "" @@ -145,7 +146,7 @@ class Role: logger.debug(self._actions) self._rc.todo = self._actions[self._rc.state] - def set_env(self, env: 'Environment'): + def set_env(self, env: "Environment"): """设置角色工作所处的环境，角色可以向环境说话，也可以通过观察接受环境消息""" self._rc.env = env @@ -192,12 +193,13 @@ class Role: self._set_state(0) return True prompt = self._get_prefix() - prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + prompt += STATE_TEMPLATE.format( + history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1 + ) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') + logger.warning(f"Invalid answer of state, {next_state=}") next_state = "0" self._set_state(int(next_state)) return True @@ -212,8 +214,12 @@ class Role: response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=type(self._rc.todo), + ) else: msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) @@ -236,7 +242,7 @@ class Role: news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: - logger.debug(f'{self._setting} observed: {news_text}') + logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) def _publish_message(self, msg): @@ -310,20 +316,15 @@ class Role: def add_to_do(self, act): self._rc.todo = act - async def think(self) -> bool: + async def think(self) -> Action: """The exported `think` function""" - has_action = await self._think() - if not has_action: - return False - if not self._rc.todo: - return False - return True + await self._think() + return self._rc.todo async def act(self) -> ActionOutput: """The exported `act` function""" msg = await self._act() - return ActionOutput(content=msg.content, - instruct_content=msg.instruct_content) + return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) @property def todo_description(self): diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index ca88fd681..031ce94c9 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -9,22 +9,34 @@ """ +import re + import aiofiles -from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.write_teaching_plan import ( + TeachingPlanRequirement, + WriteTeachingPlanPart, +) +from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger -import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, name='Lily', profile='{teaching_language} Teacher', - goal='writing a {language} teaching plan part by part', - constraints='writing in {language}', desc="", *args, **kwargs): + + def __init__( + self, + name="Lily", + profile="{teaching_language} Teacher", + goal="writing a {language} teaching plan part by part", + constraints="writing in {language}", + desc="", + *args, + **kwargs, + ): super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: @@ -54,7 +66,7 @@ class Teacher(Role): break logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") msg = await self._act() - if ret.content != '': + if ret.content != "": ret.content += "\n\n\n" ret.content += msg.content logger.info(ret.content) @@ -64,14 +76,14 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = WORKSPACE_ROOT / "teaching_plan" + pathname = CONFIG.workspace / "teaching_plan" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: - async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + async with aiofiles.open(str(pathname), mode="w", encoding="utf-8") as writer: await writer.write(content) except Exception as e: - logger.error(f'Save failed：{e}') + logger.error(f"Save failed：{e}") logger.info(f"Save to:{pathname}") @staticmethod @@ -80,8 +92,8 @@ class Teacher(Role): # Define the special characters that need to be replaced. illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' # Replace the special characters with underscores. - filename = re.sub(illegal_chars, '_', lesson_title) + ext - return re.sub(r'_+', '_', filename) + filename = re.sub(illegal_chars, "_", lesson_title) + ext + return re.sub(r"_+", "_", filename) @property def course_title(self): @@ -93,9 +105,9 @@ class Teacher(Role): if act.rsp is None: return default_title title = act.rsp.lstrip("# \n") - if '\n' in title: - ix = title.index('\n') - title = title[0: ix] + if "\n" in title: + ix = title.index("\n") + title = title[0:ix] return title return default_title diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index a63dbe5ac..c33f67a51 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -14,7 +14,6 @@ from aiohttp import ClientSession from PIL import Image, PngImagePlugin from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger config = Config() @@ -81,7 +80,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.get_workspace() / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 1245671fb..15fd08625 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -6,19 +6,20 @@ @File : mermaid.py @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -import subprocess +import asyncio from pathlib import Path -from metagpt.config import Config +# from metagpt.utils.common import check_cmd_exists +import aiofiles + +from metagpt.config import CONFIG, Config from metagpt.const import PROJECT_ROOT from metagpt.logs import logger -from metagpt.utils.common import check_cmd_exists -def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """suffix: png/svg/pdf - :param options: runtime context options, created by `Config` class object and changed in flow pipeline :param mermaid_code: mermaid code :param output_file_without_suffix: output filename :param width: @@ -27,92 +28,87 @@ def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=204 """ # Write the Mermaid code to a temporary file tmp = Path(f"{output_file_without_suffix}.mmd") - tmp.write_text(mermaid_code, encoding="utf-8") + async with aiofiles.open(tmp, "w", encoding="utf-8") as f: + await f.write(mermaid_code) + # tmp.write_text(mermaid_code, encoding="utf-8") - if check_cmd_exists("mmdc") != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") - return -1 + # if check_cmd_exists("mmdc") != 0: + # logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + # return -1 - for suffix in ["pdf", "svg", "png"]: + # for suffix in ["pdf", "svg", "png"]: + for suffix in ["png"]: output_file = f"{output_file_without_suffix}.{suffix}" # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") + cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)] - if options.get("puppeteer_config"): - subprocess.run( - [ - options.get("mmdc"), - "-p", - options.get("puppeteer_config"), - "-i", - str(tmp), - "-o", - output_file, - "-w", - str(width), - "-H", - str(height), - ] - ) - else: - subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) - return 0 - - -MMC1 = """classDiagram - class Main { - -SearchEngine search_engine - +main() str - } - class SearchEngine { - -Index index - -Ranking ranking - -Summary summary - +search(query: str) str - } - class Index { - -KnowledgeBase knowledge_base - +create_index(data: dict) - +query_index(query: str) list - } - class Ranking { - +rank_results(results: list) list - } - class Summary { - +summarize_results(results: list) str - } - class KnowledgeBase { - +update(data: dict) - +fetch_data(query: str) dict - } - Main --> SearchEngine - SearchEngine --> Index - SearchEngine --> Ranking - SearchEngine --> Summary - Index --> KnowledgeBase""" - -MMC2 = """sequenceDiagram - participant M as Main - participant SE as SearchEngine - participant I as Index - participant R as Ranking - participant S as Summary - participant KB as KnowledgeBase - M->>SE: search(query) - SE->>I: query_index(query) - I->>KB: fetch_data(query) - KB-->>I: return data - I-->>SE: return results - SE->>R: rank_results(results) - R-->>SE: return ranked_results - SE->>S: summarize_results(ranked_results) - S-->>SE: return summary - SE-->>M: return summary""" + if CONFIG.puppeteer_config: + cmds.extend(["-p", CONFIG.puppeteer_config]) + process = await asyncio.create_subprocess_exec(*cmds) + await process.wait() + return process.returncode if __name__ == "__main__": + MMC1 = """classDiagram + class Main { + -SearchEngine search_engine + +main() str + } + class SearchEngine { + -Index index + -Ranking ranking + -Summary summary + +search(query: str) str + } + class Index { + -KnowledgeBase knowledge_base + +create_index(data: dict) + +query_index(query: str) list + } + class Ranking { + +rank_results(results: list) list + } + class Summary { + +summarize_results(results: list) str + } + class KnowledgeBase { + +update(data: dict) + +fetch_data(query: str) dict + } + Main --> SearchEngine + SearchEngine --> Index + SearchEngine --> Ranking + SearchEngine --> Summary + Index --> KnowledgeBase""" + + MMC2 = """sequenceDiagram + participant M as Main + participant SE as SearchEngine + participant I as Index + participant R as Ranking + participant S as Summary + participant KB as KnowledgeBase + M->>SE: search(query) + SE->>I: query_index(query) + I->>KB: fetch_data(query) + KB-->>I: return data + I-->>SE: return results + SE->>R: rank_results(results) + R-->>SE: return ranked_results + SE->>S: summarize_results(ranked_results) + S-->>SE: return summary + SE-->>M: return summary""" + conf = Config() - mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1, - output_file_without_suffix=PROJECT_ROOT / "tmp/1.png") - mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2, - output_file_without_suffix=PROJECT_ROOT / "tmp/2.png") + asyncio.run( + mermaid_to_file( + options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png" + ) + ) + asyncio.run( + mermaid_to_file( + options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png" + ) + ) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..8e9660e36 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,7 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -214,7 +214,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 0a2ca4071..b7f94a19c 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -8,11 +8,8 @@ @Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ import asyncio -import sys -from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.tools.azure_tts import AzureTTS @@ -28,15 +25,13 @@ def test_azure_tts(): “Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.” """ - path = WORKSPACE_ROOT / "tts" + path = CONFIG.workspace / "tts" path.mkdir(exist_ok=True, parents=True) filename = path / "girl.wav" loop = asyncio.new_event_loop() - v = loop.create_task(azure_tts.synthesize_speech( - lang="zh-CN", - voice="zh-CN-XiaomoNeural", - text=text, - output_file=str(filename))) + v = loop.create_task( + azure_tts.synthesize_speech(lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, output_file=str(filename)) + ) result = loop.run_until_complete(v) print(result) @@ -45,5 +40,5 @@ def test_azure_tts(): # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 -if __name__ == '__main__': +if __name__ == "__main__": test_azure_tts() diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..89c97f5e8 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,8 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine def test_sd_engine_init(): @@ -21,5 +22,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True From 43dda1edafc25df7c99c76efa2b31486fd75e710 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 30 Aug 2023 11:55:54 +0800 Subject: [PATCH 02/11] fix options error --- metagpt/actions/project_management.py | 3 ++- .../tools/web_browser_engine_playwright.py | 20 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 55e7cbcb5..1062f8984 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -109,7 +109,8 @@ class WriteTasks(Action): async def _save(self, rsp): file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" - file_path.write_text(rsp.content) + async with aiofiles.open(file_path, "w") as f: + await f.write(rsp.content) # Write requirements.txt requirements_path = CONFIG.workspace / "requirements.txt" diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index 199f8a0d1..8eecc4f40 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -8,11 +8,11 @@ from __future__ import annotations import asyncio import sys from pathlib import Path -from typing import Literal, Dict +from typing import Literal from playwright.async_api import async_playwright -from metagpt.config import Config +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.parse_html import WebPage @@ -28,20 +28,18 @@ class PlaywrightWrapper: def __init__( self, - options: Dict, browser_type: Literal["chromium", "firefox", "webkit"] | None = None, launch_kwargs: dict | None = None, **kwargs, ) -> None: - self.options = options if browser_type is None: - browser_type = options.get("playwright_browser_type") + browser_type = CONFIG.playwright_browser_type self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if options.get("global_proxy") and "proxy" not in launch_kwargs: + if CONFIG.global_proxy and "proxy" not in launch_kwargs: args = launch_kwargs.get("args", []) if not any(str.startswith(i, "--proxy-server=") for i in args): - launch_kwargs["proxy"] = {"server": options.get("global_proxy")} + launch_kwargs["proxy"] = {"server": CONFIG.global_proxy} self.launch_kwargs = launch_kwargs context_kwargs = {} if "ignore_https_errors" in kwargs: @@ -81,8 +79,8 @@ class PlaywrightWrapper: executable_path = Path(browser_type.executable_path) if not executable_path.exists() and "executable_path" not in self.launch_kwargs: kwargs = {} - if self.options.get("global_proxy"): - kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")} + if CONFIG.global_proxy: + kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy} await _install_browsers(self.browser_type, **kwargs) if self._has_run_precheck: @@ -150,8 +148,6 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs): - return await PlaywrightWrapper(options=Config().runtime_options, - browser_type=browser_type, - **kwargs).run(url, *urls) + return await PlaywrightWrapper(browser_type=browser_type, **kwargs).run(url, *urls) fire.Fire(main) From 9428c256caf1f16971216c7a3e4b66603bf8a825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 17:55:13 +0800 Subject: [PATCH 03/11] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 33 +++++++++++++++++++ .../metagpt/provider/test_metagpt_llm_api.py | 17 ++++++++++ 2 files changed, 50 insertions(+) create mode 100644 metagpt/provider/metagpt_llm_api.py create mode 100644 tests/metagpt/provider/test_metagpt_llm_api.py diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py new file mode 100644 index 000000000..bfd003fff --- /dev/null +++ b/metagpt/provider/metagpt_llm_api.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/30 +@Author : mashenquan +@File : metagpt_llm_api.py +@Desc : MetaGPT LLM related APIs +""" + +import openai + +from metagpt.config import CONFIG +from metagpt.provider import OpenAIGPTAPI +from metagpt.provider.openai_api import RateLimiter + + +class MetaGPTLLMAPI(OpenAIGPTAPI): + """MetaGPT LLM api""" + + def __init__(self): + self.__init_openai(CONFIG) + self.llm = openai + self.model = CONFIG.METAGPT_API_MODEL + self.auto_max_tokens = False + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_openai(self, config): + openai.api_key = CONFIG.METAGPT_API_KEY + if config.openai_api_base: + openai.api_base = CONFIG.METAGPT_API_BASE + if config.openai_api_type: + openai.api_type = CONFIG.METAGPT_API_TYPE + openai.api_version = CONFIG.METAGPT_API_VERSION + self.rpm = int(config.get("RPM", 10)) diff --git a/tests/metagpt/provider/test_metagpt_llm_api.py b/tests/metagpt/provider/test_metagpt_llm_api.py new file mode 100644 index 000000000..9c8356ca6 --- /dev/null +++ b/tests/metagpt/provider/test_metagpt_llm_api.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/30 +@Author : mashenquan +@File : test_metagpt_llm_api.py +""" +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI + + +def test_metagpt(): + llm = MetaGPTLLMAPI() + assert llm + + +if __name__ == "__main__": + test_metagpt() From 09fdb9d1ae1e5d0ab5f6a9c4571cff6bb265089f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:21:36 +0800 Subject: [PATCH 04/11] feat: +metagpt llm --- metagpt/const.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 9e7462da6..e792ff35a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -14,9 +14,11 @@ def get_project_root(): """逐级向上寻找项目根目录""" current_path = Path.cwd() while True: - if (current_path / '.git').exists() or \ - (current_path / '.project_root').exists() or \ - (current_path / '.gitignore').exists(): + if ( + (current_path / ".git").exists() + or (current_path / ".project_root").exists() + or (current_path / ".gitignore").exists() + ): return current_path parent_path = current_path.parent if parent_path == current_path: @@ -25,15 +27,15 @@ def get_project_root(): PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / 'data' -WORKSPACE_ROOT = PROJECT_ROOT / 'workspace' -PROMPT_PATH = PROJECT_ROOT / 'metagpt/prompts' -UT_PATH = PROJECT_ROOT / 'data/ut' +DATA_PATH = PROJECT_ROOT / "data" +WORKSPACE_ROOT = PROJECT_ROOT / "workspace" +PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" +UT_PATH = PROJECT_ROOT / "data/ut" SWAGGER_PATH = UT_PATH / "files/api/" UT_PY_PATH = UT_PATH / "files/ut/" API_QUESTIONS_PATH = UT_PATH / "files/question/" YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / 'tmp' +TMP = PROJECT_ROOT / "tmp" RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 @@ -43,4 +45,12 @@ DEFAULT_LANGUAGE = "English" DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 BRAIN_MEMORY = "BRAIN_MEMORY" -SKILL_PATH = "SKILL_PATH" \ No newline at end of file +SKILL_PATH = "SKILL_PATH" +SERPER_API_KEY = "SERPER_API_KEY" + +# MetaGPT LLM key defines +METAGPT_API_MODEL = "METAGPT_API_MODEL" +METAGPT_API_KEY = "METAGPT_API_KEY" +METAGPT_API_BASE = "METAGPT_API_BASE" +METAGPT_API_TYPE = "METAGPT_API_TYPE" +METAGPT_API_VERSION = "METAGPT_API_VERSION" From f65b959d5277053ddffebdc3fdc5e8a11af9c6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:23:29 +0800 Subject: [PATCH 05/11] feat: +metagpt llm --- metagpt/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/const.py b/metagpt/const.py index e792ff35a..f2f1b4837 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -48,7 +48,7 @@ BRAIN_MEMORY = "BRAIN_MEMORY" SKILL_PATH = "SKILL_PATH" SERPER_API_KEY = "SERPER_API_KEY" -# MetaGPT LLM key defines +# Key Definitions for MetaGPT LLM METAGPT_API_MODEL = "METAGPT_API_MODEL" METAGPT_API_KEY = "METAGPT_API_KEY" METAGPT_API_BASE = "METAGPT_API_BASE" From 39e2e1d8a01be2696b3319f0b7c5794af7a650f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:25:10 +0800 Subject: [PATCH 06/11] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index bfd003fff..78a9e44b1 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -25,9 +25,9 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init_openai(self, config): openai.api_key = CONFIG.METAGPT_API_KEY - if config.openai_api_base: + if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE - if config.openai_api_type: + if CONFIG.METAGPT_API_TYPE: openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(config.get("RPM", 10)) From 4e92206301a43edfd6e777a1bff43e99acb884dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:26:52 +0800 Subject: [PATCH 07/11] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 78a9e44b1..bb8749e82 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -17,17 +17,17 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.__init_openai(CONFIG) + self.__init_openai() self.llm = openai self.model = CONFIG.METAGPT_API_MODEL self.auto_max_tokens = False RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): + def __init_openai(self): openai.api_key = CONFIG.METAGPT_API_KEY if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE if CONFIG.METAGPT_API_TYPE: openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION - self.rpm = int(config.get("RPM", 10)) + self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 From 01bdc2c90bcb8056f854c0560b6df7fa1137f43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:28:13 +0800 Subject: [PATCH 08/11] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index bb8749e82..c27e7132d 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -23,7 +23,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): self.auto_max_tokens = False RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self): + def __init_openai(self, *args, **kwargs): openai.api_key = CONFIG.METAGPT_API_KEY if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE From d304e008a0d2d43ef538e22b821fb09568366272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 14:36:23 +0800 Subject: [PATCH 09/11] feat: +log --- metagpt/provider/base_gpt_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f1590a77c..af0cf2ec0 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -15,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -46,9 +47,9 @@ class BaseGPTAPI(BaseChatbot): rsp = await self.acompletion_text(message, stream=True) except Exception as e: logger.exception(f"{e}") + logger.info(f"ask:{msg}, error:{e}") raise e - logger.debug(message) - # logger.debug(rsp) + logger.info(f"ask:{msg}, anwser:{rsp}") return rsp def _extract_assistant_rsp(self, context): @@ -115,7 +116,7 @@ class BaseGPTAPI(BaseChatbot): def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" From 478139c8dc2286d8e3db722145626a408cba4159 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Fri, 1 Sep 2023 21:21:47 +0800 Subject: [PATCH 10/11] feature: aioboto3 client --- config/config.yaml | 8 ++- metagpt/utils/s3.py | 127 +++++++++++++++++++++++++++++++++ requirements.txt | 4 +- tests/conftest.py | 7 +- tests/metagpt/utils/test_s3.py | 55 ++++++++++++++ 5 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 metagpt/utils/s3.py create mode 100644 tests/metagpt/utils/test_s3.py diff --git a/config/config.yaml b/config/config.yaml index 88cca08e5..7c3d212f6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -77,4 +77,10 @@ MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### Meta Models -#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL \ No newline at end of file +#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL + +### S3 config +S3: + access_key: "YOUR_S3_ACCESS_KEY" + secret_key: "YOUR_S3_SECRET_KEY" + endpoint_url: "YOUR_S3_ENDPOINT_URL" \ No newline at end of file diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py new file mode 100644 index 000000000..2b4b8cb5f --- /dev/null +++ b/metagpt/utils/s3.py @@ -0,0 +1,127 @@ + +from typing import Optional + +import aioboto3 +from metagpt.logs import logger +from metagpt.config import Config + + +class S3: + """A class for interacting with Amazon S3 storage.""" + + def __init__(self): + self.session = aioboto3.Session() + self.s3_config = Config().get("S3") + self.auth_config = { + "service_name": "s3", + "aws_access_key_id": self.s3_config["access_key"], + "aws_secret_access_key": self.s3_config["secret_key"], + "endpoint_url": self.s3_config["endpoint_url"] + } + + async def upload_file( + self, + bucket: str, + local_path: str, + object_name: str, + ) -> None: + """Upload a file from the local path to the specified path of the storage bucket specified in s3. + + Args: + bucket: The name of the S3 storage bucket. + local_path: The local file path, including the file name. + object_name: The complete path of the uploaded file to be stored in S3, including the file name. + + Raises: + Exception: If an error occurs during the upload process, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + with open(local_path, "rb") as file: + await client.put_object(Body=file, Bucket=bucket, Key=object_name) + logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.") + except Exception as e: + logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}") + raise e + + async def get_object_url( + self, + bucket: str, + object_name: str, + ) -> str: + """Get the URL for a downloadable or preview file stored in the specified S3 bucket. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + + Returns: + The URL for the downloadable or preview file. + + Raises: + Exception: If an error occurs while retrieving the URL, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + file = await client.get_object(Bucket=bucket, Key=object_name) + return str(file["Body"].url) + except Exception as e: + logger.error(f"Failed to get the url for a downloadable or preview file: {e}") + raise e + + async def get_object( + self, + bucket: str, + object_name: str, + ) -> bytes: + """Get the binary data of a file stored in the specified S3 bucket. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + + Returns: + The binary data of the requested file. + + Raises: + Exception: If an error occurs while retrieving the file data, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + s3_object = await client.get_object(Bucket=bucket, Key=object_name) + return await s3_object["Body"].read() + except Exception as e: + logger.error(f"Failed to get the binary data of the file: {e}") + raise e + + async def download_file( + self, + bucket: str, + object_name: str, + local_path: str, + chunk_size: Optional[int] = 128 * 1024 + ) -> None: + """Download an S3 object to a local file. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + local_path: The local file path where the S3 object will be downloaded. + chunk_size: The size of data chunks to read and write at a time. Default is 128 KB. + + Raises: + Exception: If an error occurs during the download process, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + s3_object = await client.get_object(Bucket=bucket, Key=object_name) + stream = s3_object["Body"] + with open(local_path, 'wb') as local_file: + while True: + file_data = await stream.read(chunk_size) + if not file_data: + break + local_file.write(file_data) + except Exception as e: + logger.error(f"Failed to download the file from S3: {e}") + raise e \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ca7fcbfda..2e5112aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,6 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.31.0 \ No newline at end of file +azure-cognitiveservices-speech==1.31.0 +aioboto3~=11.3.0 +pytest-asyncio~=0.21.1 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 8f5069bbe..0bc17bd6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ from unittest.mock import Mock import pytest -import pytest_asyncio from metagpt.config import Config from metagpt.logs import logger @@ -17,6 +16,8 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio import re +from metagpt.utils.s3 import S3 + class Context: def __init__(self): @@ -74,3 +75,7 @@ def proxy(): @pytest.fixture(scope="session", autouse=True) def init_config(): Config() + +@pytest.fixture(scope="session", autouse=True) +def s3(): + return S3() diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py new file mode 100644 index 000000000..760a976b0 --- /dev/null +++ b/tests/metagpt/utils/test_s3.py @@ -0,0 +1,55 @@ +import os +import pytest + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "local_path", "object_name"], + [ + ( + "agent-store", + "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", + "ui-designer/2023-09-01/1.png" + ) + ] +) +async def test_upload_file(s3, bucket, local_path, object_name): + await s3.upload_file(bucket=bucket, local_path=local_path, object_name=object_name) + s3_object = await s3.get_object(bucket=bucket, object_name=object_name) + assert s3_object + assert isinstance(s3_object, bytes) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "object_name"], + [("agent-store", "ui-designer/2023-09-01/1.png")] +) +async def test_get_object_url(s3, bucket, object_name): + url = await s3.get_object_url(bucket=bucket, object_name=object_name) + assert bucket in url + assert object_name in url + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "object_name"], + [("agent-store", "ui-designer/2023-09-01/1.png")] +) +async def test_get_object(s3, bucket, object_name): + s3_object = await s3.get_object(bucket=bucket, object_name=object_name) + assert s3_object + assert isinstance(s3_object, bytes) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "local_path", "object_name"], + [ + ( + "agent-store", + "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", + "ui-designer/2023-09-01/1.png" + ) + ] +) +async def test_download_file(s3, bucket, local_path, object_name): + await s3.download_file(bucket=bucket, object_name=object_name, local_path=local_path) + assert os.path.exists(local_path) \ No newline at end of file From bfd8ed69e8676e204e60d94d25e52605d528f8b5 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Sat, 2 Sep 2023 10:55:38 +0800 Subject: [PATCH 11/11] update: delete pytest code --- requirements.txt | 3 +- tests/conftest.py | 5 ---- tests/metagpt/utils/test_s3.py | 55 ---------------------------------- 3 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 tests/metagpt/utils/test_s3.py diff --git a/requirements.txt b/requirements.txt index 2e5112aba..5daf710c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,5 +41,4 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 -aioboto3~=11.3.0 -pytest-asyncio~=0.21.1 \ No newline at end of file +aioboto3~=11.3.0 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 0bc17bd6a..98b45de7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,6 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio import re -from metagpt.utils.s3 import S3 - class Context: def __init__(self): @@ -76,6 +74,3 @@ def proxy(): def init_config(): Config() -@pytest.fixture(scope="session", autouse=True) -def s3(): - return S3() diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py deleted file mode 100644 index 760a976b0..000000000 --- a/tests/metagpt/utils/test_s3.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import pytest - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "local_path", "object_name"], - [ - ( - "agent-store", - "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", - "ui-designer/2023-09-01/1.png" - ) - ] -) -async def test_upload_file(s3, bucket, local_path, object_name): - await s3.upload_file(bucket=bucket, local_path=local_path, object_name=object_name) - s3_object = await s3.get_object(bucket=bucket, object_name=object_name) - assert s3_object - assert isinstance(s3_object, bytes) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "object_name"], - [("agent-store", "ui-designer/2023-09-01/1.png")] -) -async def test_get_object_url(s3, bucket, object_name): - url = await s3.get_object_url(bucket=bucket, object_name=object_name) - assert bucket in url - assert object_name in url - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "object_name"], - [("agent-store", "ui-designer/2023-09-01/1.png")] -) -async def test_get_object(s3, bucket, object_name): - s3_object = await s3.get_object(bucket=bucket, object_name=object_name) - assert s3_object - assert isinstance(s3_object, bytes) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "local_path", "object_name"], - [ - ( - "agent-store", - "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", - "ui-designer/2023-09-01/1.png" - ) - ] -) -async def test_download_file(s3, bucket, local_path, object_name): - await s3.download_file(bucket=bucket, object_name=object_name, local_path=local_path) - assert os.path.exists(local_path) \ No newline at end of file