apunkt/MetaGPT
1
0
Fork 0
mirror of https://github.com/FoundationAgents/MetaGPT.git synced 2026-05-09 15:52:38 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: merge send18:dev

Browse source
This commit is contained in:
莘权 马 2023-12-14 15:06:04 +08:00
commit 7effe7f74c
92 changed files with 4830 additions and 302 deletions
Download patch file Download diff file Expand all files Collapse all files

15
metagpt/__init__.py
View file

@ -1,7 +1,22 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
<<<<<<< HEAD
# @Time : 2023/4/24 22:26
# @Author : alexanderwu
# @File : __init__.py
from metagpt import _compat as _ # noqa: F401
=======
"""
@Time : 2023/4/24 22:26
@Author : alexanderwu
@File : __init__.py
@Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects.
"""
from metagpt.schema import Message
__all__ = [
"Message",
]
>>>>>>> send18/dev

14
metagpt/actions/action.py
View file

@ -4,22 +4,26 @@
@Time : 2023/5/11 14:43
@Author : alexanderwu
@File : action.py
@Modified By: mashenquan, 2023/8/20. Add function return annotations.
@Modified By: mashenquan, 2023/9/8. Replace LLM with LLMFactory
"""
import re
from __future__ import annotations
from abc import ABC
from typing import Optional
from tenacity import retry, stop_after_attempt, wait_random_exponential
from metagpt.actions.action_output import ActionOutput
from metagpt.llm import LLM
from metagpt.logs import logger
from metagpt.utils.common import OutputParser
from metagpt.utils.custom_decoder import CustomDecoder
from metagpt.logs import logger
from metagpt.provider.base_gpt_api import BaseGPTAPI
from metagpt.utils.common import OutputParser
class Action(ABC):
def __init__(self, name: str = "", context=None, llm: LLM = None):
def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None):
self.name: str = name
if llm is None:
llm = LLM()
@ -88,6 +92,6 @@ class Action(ABC):
instruct_content = output_class(**parsed_data)
return ActionOutput(content, instruct_content)
async def run(self, *args, **kwargs):
async def run(self, *args, **kwargs) -> str | ActionOutput | None:
"""Run action"""
raise NotImplementedError("The run method should be implemented in a subclass.")

7
metagpt/actions/action_output.py
View file

@ -4,18 +4,19 @@
@Time : 2023/7/11 10:03
@Author : chengmaoyu
@File : action_output
@Modified By: mashenquan, 2023/8/20. Allow 'instruct_content' to be blank.
"""
from typing import Dict, Type
from typing import Dict, Type, Optional
from pydantic import BaseModel, create_model, root_validator, validator
class ActionOutput:
content: str
instruct_content: BaseModel
instruct_content: Optional[BaseModel] = None
def __init__(self, content: str, instruct_content: BaseModel):
def __init__(self, content: str, instruct_content: BaseModel=None):
self.content = content
self.instruct_content = instruct_content

45
metagpt/actions/azure_tts.py
View file

@ -1,45 +0,0 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/6/9 22:22
@Author : Leo Xiao
@File : azure_tts.py
"""
from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer
from metagpt.actions.action import Action
from metagpt.config import Config
class AzureTTS(Action):
def __init__(self, name, context=None, llm=None):
super().__init__(name, context, llm)
self.config = Config()
# Parameters reference: https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles
def synthesize_speech(self, lang, voice, role, text, output_file):
subscription_key = self.config.get("AZURE_TTS_SUBSCRIPTION_KEY")
region = self.config.get("AZURE_TTS_REGION")
speech_config = SpeechConfig(subscription=subscription_key, region=region)
speech_config.speech_synthesis_voice_name = voice
audio_config = AudioConfig(filename=output_file)
synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config)
# if voice=="zh-CN-YunxiNeural":
ssml_string = f"""
<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' xml:lang='{lang}' xmlns:mstts='http://www.w3.org/2001/mstts'>
<voice name='{voice}'>
<mstts:express-as style='affectionate' role='{role}'>
{text}
</mstts:express-as>
</voice>
</speak>
"""
synthesizer.speak_ssml_async(ssml_string).get()
if __name__ == "__main__":
azure_tts = AzureTTS("azure_tts")
azure_tts.synthesize_speech("zh-CN", "zh-CN-YunxiNeural", "Boy", "Hello, I am Kaka", "output.wav")

36
metagpt/actions/design_api.py
View file

@ -4,6 +4,7 @@
@Time : 2023/5/11 19:26
@Author : alexanderwu
@File : design_api.py
<<<<<<< HEAD
@Modified By: mashenquan, 2023/11/27.
1. According to Section 2.2.3.1 of RFC 135, replace file data in the message with the file name.
2. According to the design in Section 2.2.3.5.3 of RFC 135, add incremental iteration functionality.
@ -22,6 +23,16 @@ from metagpt.const import (
SYSTEM_DESIGN_FILE_REPO,
SYSTEM_DESIGN_PDF_FILE_REPO,
)
=======
@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class.
"""
from typing import List
import aiofiles
from metagpt.actions import Action
from metagpt.config import CONFIG
>>>>>>> send18/dev
from metagpt.logs import logger
from metagpt.schema import Document, Documents
from metagpt.utils.file_repository import FileRepository
@ -197,6 +208,7 @@ class WriteDesign(Action):
"clearly and in detail."
)
<<<<<<< HEAD
async def run(self, with_messages, format=CONFIG.prompt_format):
# Use `git diff` to identify which PRD documents have been modified in the `docs/prds` directory.
prds_file_repo = CONFIG.git_repo.new_file_repository(PRDS_FILE_REPO)
@ -232,6 +244,30 @@ class WriteDesign(Action):
format_example = format_example.format(project_name=CONFIG.project_name)
prompt = prompt_template.format(context=context, format_example=format_example)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING, format=format)
=======
async def _save_system_design(self, docs_path, resources_path, content):
data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content)
seq_flow = CodeParser.parse_code(block="Program call flow", text=content)
await mermaid_to_file(data_api_design, resources_path / "data_api_design")
await mermaid_to_file(seq_flow, resources_path / "seq_flow")
system_design_file = docs_path / "system_design.md"
logger.info(f"Saving System Designs to {system_design_file}")
async with aiofiles.open(system_design_file, "w") as f:
await f.write(content)
async def _save(self, system_design: str):
workspace = CONFIG.workspace
docs_path = workspace / "docs"
resources_path = workspace / "resources"
docs_path.mkdir(parents=True, exist_ok=True)
resources_path.mkdir(parents=True, exist_ok=True)
await self._save_system_design(docs_path, resources_path, system_design)
async def run(self, context, **kwargs):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING)
await self._save(system_design.content)
>>>>>>> send18/dev
return system_design
async def _merge(self, prd_doc, system_design_doc, format=CONFIG.prompt_format):

38
metagpt/actions/project_management.py
View file

@ -4,14 +4,19 @@
@Time : 2023/5/11 19:12
@Author : alexanderwu
@File : project_management.py
<<<<<<< HEAD
@Modified By: mashenquan, 2023/11/27.
1. Divide the context into three components: legacy code, unit test code, and console log.
2. Move the document storage operations related to WritePRD from the save operation of WriteDesign.
3. According to the design in Section 2.2.3.5.4 of RFC 135, add incremental iteration functionality.
=======
@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class.
>>>>>>> send18/dev
"""
import json
from typing import List
<<<<<<< HEAD
from metagpt.actions import ActionOutput
from metagpt.actions.action import Action
from metagpt.config import CONFIG
@ -86,6 +91,14 @@ and only output the json inside this tag, nothing else
},
"markdown": {
"PROMPT_TEMPLATE": """
=======
import aiofiles
from metagpt.actions.action import Action
from metagpt.config import CONFIG
PROMPT_TEMPLATE = """
>>>>>>> send18/dev
# Context
{context}
@ -108,7 +121,11 @@ Attention: Use '##' to split sections, not '#', and '## <SECTION_NAME>' SHOULD W
## Shared Knowledge: Anything that should be public like utils' functions, config's variables details that should make clear first.
<<<<<<< HEAD
## Anything UNCLEAR: Provide as Plain text. Try to clarify it. For example, don't forget a main entry. don't forget to init 3rd party libs.
=======
"""
>>>>>>> send18/dev
""",
"FORMAT_EXAMPLE": '''
@ -180,6 +197,7 @@ MERGE_PROMPT = """
# Context
{context}
<<<<<<< HEAD
## Old Tasks
{old_tasks}
-----
@ -210,10 +228,13 @@ and only output the json inside this tag, nothing else
"""
=======
>>>>>>> send18/dev
class WriteTasks(Action):
def __init__(self, name="CreateTasks", context=None, llm=None):
super().__init__(name, context, llm)
<<<<<<< HEAD
async def run(self, with_messages, format=CONFIG.prompt_format):
system_design_file_repo = CONFIG.git_repo.new_file_repository(SYSTEM_DESIGN_FILE_REPO)
changed_system_designs = system_design_file_repo.changed_files
@ -265,6 +286,23 @@ class WriteTasks(Action):
prompt_template, format_example = get_template(templates, format)
prompt = prompt_template.format(context=context, format_example=format_example)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING, format=format)
=======
async def _save(self, rsp):
file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md"
async with aiofiles.open(file_path, "w") as f:
await f.write(rsp.content)
# Write requirements.txt
requirements_path = CONFIG.workspace / "requirements.txt"
async with aiofiles.open(requirements_path, "w") as f:
await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n'))
async def run(self, context, **kwargs):
prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE)
rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING)
await self._save(rsp)
>>>>>>> send18/dev
return rsp
async def _merge(self, system_design_doc, task_doc, format=CONFIG.prompt_format) -> Document:

6
metagpt/actions/search_and_summarize.py
View file

@ -4,11 +4,12 @@
@Time : 2023/5/23 17:26
@Author : alexanderwu
@File : search_google.py
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
import pydantic
from metagpt.actions import Action
from metagpt.config import Config
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.schema import Message
from metagpt.tools.search_engine import SearchEngine
@ -102,8 +103,7 @@ You are a member of a professional butler team and will provide helpful suggesti
class SearchAndSummarize(Action):
def __init__(self, name="", context=None, llm=None, engine=None, search_func=None):
self.config = Config()
self.engine = engine or self.config.search_engine
self.engine = engine or CONFIG.search_engine
try:
self.search_engine = SearchEngine(self.engine, run_func=search_func)

109
metagpt/actions/skill_action.py Normal file
View file

@ -0,0 +1,109 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/28
@Author : mashenquan
@File : skill_action.py
@Desc : Call learned skill
"""
from __future__ import annotations
import ast
import importlib
import traceback
from copy import deepcopy
from metagpt.actions import Action, ActionOutput
from metagpt.learn.skill_loader import Skill
from metagpt.logs import logger
class ArgumentsParingAction(Action):
def __init__(self, last_talk: str, skill: Skill, context=None, llm=None, **kwargs):
super(ArgumentsParingAction, self).__init__(name="", context=context, llm=llm)
self.skill = skill
self.ask = last_talk
self.rsp = None
self.args = None
@property
def prompt(self):
prompt = f"{self.skill.name} function parameters description:\n"
for k, v in self.skill.arguments.items():
prompt += f"parameter `{k}`: {v}\n"
prompt += "\n"
prompt += "Examples:\n"
for e in self.skill.examples:
prompt += f"If want you to do `{e.ask}`, return `{e.answer}` brief and clear.\n"
prompt += f"\nNow I want you to do `{self.ask}`, return in examples format above, brief and clear."
return prompt
async def run(self, *args, **kwargs) -> ActionOutput:
prompt = self.prompt
rsp = await self.llm.aask(msg=prompt, system_msgs=[])
logger.debug(f"SKILL:{prompt}\n, RESULT:{rsp}")
self.args = ArgumentsParingAction.parse_arguments(skill_name=self.skill.name, txt=rsp)
self.rsp = ActionOutput(content=rsp)
return self.rsp
@staticmethod
def parse_arguments(skill_name, txt) -> dict:
prefix = skill_name + "("
if prefix not in txt:
logger.error(f"{skill_name} not in {txt}")
return None
if ")" not in txt:
logger.error(f"')' not in {txt}")
return None
begin_ix = txt.find(prefix)
end_ix = txt.rfind(")")
args_txt = txt[begin_ix + len(prefix) : end_ix]
logger.info(args_txt)
fake_expression = f"dict({args_txt})"
parsed_expression = ast.parse(fake_expression, mode="eval")
args = {}
for keyword in parsed_expression.body.keywords:
key = keyword.arg
value = ast.literal_eval(keyword.value)
args[key] = value
return args
class SkillAction(Action):
def __init__(self, skill: Skill, args: dict, context=None, llm=None, **kwargs):
super(SkillAction, self).__init__(name="", context=context, llm=llm)
self._skill = skill
self._args = args
self.rsp = None
async def run(self, *args, **kwargs) -> str | ActionOutput | None:
"""Run action"""
options = deepcopy(kwargs)
if self._args:
for k in self._args.keys():
if k in options:
options.pop(k)
try:
self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **options)
except Exception as e:
logger.exception(f"{e}, traceback:{traceback.format_exc()}")
self.rsp = f"Error: {e}"
return ActionOutput(content=self.rsp, instruct_content=self._skill.json())
@staticmethod
async def find_and_call_function(function_name, args, **kwargs):
try:
module = importlib.import_module("metagpt.learn")
function = getattr(module, function_name)
# 调用函数并返回结果
result = await function(**args, **kwargs)
return result
except (ModuleNotFoundError, AttributeError):
logger.error(f"{function_name} not found")
return None
if __name__ == "__main__":
ArgumentsParingAction.parse_arguments(
skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`'
)

169
metagpt/actions/talk_action.py Normal file
View file

@ -0,0 +1,169 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/28
@Author : mashenquan
@File : talk_action.py
@Desc : Act as it’s a talk
"""
import json
from metagpt.actions import Action, ActionOutput
from metagpt.config import CONFIG
from metagpt.const import DEFAULT_LANGUAGE
from metagpt.llm import LLMType
from metagpt.logs import logger
class TalkAction(Action):
def __init__(self, name: str = "", talk="", history_summary="", knowledge="", context=None, llm=None, **kwargs):
context = context or {}
context["talk"] = talk
context["history_summery"] = history_summary
context["knowledge"] = knowledge
super(TalkAction, self).__init__(name=name, context=context, llm=llm)
self._talk = talk
self._history_summary = history_summary
self._knowledge = knowledge
self._rsp = None
@property
def prompt(self):
prompt = ""
if CONFIG.agent_description:
prompt = (
f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, "
f"maintaining the character's persona and habits. When faced with unrelated questions, playfully "
f"decline to answer without revealing your AI nature to preserve the character's image.\n\n"
)
prompt += f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else ""
prompt += f"{self._history_summary}\n\n"
prompt += (
"If the information is insufficient, you can search in the historical conversation or knowledge above.\n"
)
language = CONFIG.language or DEFAULT_LANGUAGE
prompt += (
f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n "
f"{self._talk}"
)
logger.debug(f"PROMPT: {prompt}")
return prompt
@property
def prompt_gpt4(self):
kvs = {
"{role}": CONFIG.agent_description or "",
"{history}": self._history_summary or "",
"{knowledge}": self._knowledge or "",
"{language}": CONFIG.language or DEFAULT_LANGUAGE,
"{ask}": self._talk,
}
prompt = TalkAction.__FORMATION_LOOSE__
for k, v in kvs.items():
prompt = prompt.replace(k, v)
logger.info(f"PROMPT: {prompt}")
return prompt
async def run_old(self, *args, **kwargs) -> ActionOutput:
prompt = self.prompt
rsp = await self.llm.aask(msg=prompt, system_msgs=[])
logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n")
self._rsp = ActionOutput(content=rsp)
return self._rsp
@property
def aask_args(self):
language = CONFIG.language or DEFAULT_LANGUAGE
system_msgs = [
f"You are {CONFIG.agent_description}.",
"Your responses should align with the role-play agreement, "
"maintaining the character's persona and habits. When faced with unrelated questions, playfully "
"decline to answer without revealing your AI nature to preserve the character's image.",
"If the information is insufficient, you can search in the context or knowledge.",
f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.",
]
format_msgs = []
if self._knowledge:
format_msgs.append({"role": "assistant", "content": self._knowledge})
if self._history_summary:
if CONFIG.LLM_TYPE == LLMType.METAGPT.value:
format_msgs.extend(json.loads(self._history_summary))
else:
format_msgs.append({"role": "assistant", "content": self._history_summary})
return self._talk, format_msgs, system_msgs
async def run(self, *args, **kwargs) -> ActionOutput:
msg, format_msgs, system_msgs = self.aask_args
rsp = await self.llm.aask(msg=msg, format_msgs=format_msgs, system_msgs=system_msgs)
self._rsp = ActionOutput(content=rsp)
return self._rsp
__FORMATION__ = """Formation: "Capacity and role" defines the role you are currently playing;
"[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation;
"[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses;
"Statement" defines the work detail you need to complete at this stage;
"[ASK_BEGIN]" and [ASK_END] tags enclose the questions;
"Constraint" defines the conditions that your responses must comply with.
"Personality" defines your language style。
"Insight" provides a deeper understanding of the characters' inner traits.
"Initial" defines the initial setup of a character.
Capacity and role: {role}
Statement: Your responses should align with the role-play agreement, maintaining the
character's persona and habits. When faced with unrelated questions, playfully decline to answer without revealing
your AI nature to preserve the character's image.
[HISTORY_BEGIN]
{history}
[HISTORY_END]
[KNOWLEDGE_BEGIN]
{knowledge}
[KNOWLEDGE_END]
Statement: If the information is insufficient, you can search in the historical conversation or knowledge.
Statement: Unless you are a language professional, answer the following questions strictly in {language}
, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]"
, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses.
{ask}
"""
__FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing;
"[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation;
"[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses;
"Statement" defines the work detail you need to complete at this stage;
"Constraint" defines the conditions that your responses must comply with.
"Personality" defines your language style。
"Insight" provides a deeper understanding of the characters' inner traits.
"Initial" defines the initial setup of a character.
Capacity and role: {role}
Statement: Your responses should maintaining the character's persona and habits. When faced with unrelated questions
, playfully decline to answer without revealing your AI nature to preserve the character's image.
[HISTORY_BEGIN]
{history}
[HISTORY_END]
[KNOWLEDGE_BEGIN]
{knowledge}
[KNOWLEDGE_END]
Statement: If the information is insufficient, you can search in the historical conversation or knowledge.
Statement: Unless you are a language professional, answer the following questions strictly in {language}
, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]"
, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses.
{ask}
"""

23
metagpt/actions/write_code.py
View file

@ -14,6 +14,7 @@
3. Encapsulate the input of RunCode into RunCodeContext and encapsulate the output of RunCode into
RunCodeResult to standardize and unify parameter passing between WriteCode, RunCode, and DebugError.
"""
<<<<<<< HEAD
import json
from tenacity import retry, stop_after_attempt, wait_random_exponential
@ -22,10 +23,18 @@ from metagpt.actions.action import Action
from metagpt.config import CONFIG
from metagpt.const import CODE_SUMMARIES_FILE_REPO, TEST_OUTPUTS_FILE_REPO, TASK_FILE_REPO, BUGFIX_FILENAME, \
DOCS_FILE_REPO
=======
from tenacity import retry, stop_after_attempt, wait_fixed
from metagpt.actions.action import Action
>>>>>>> send18/dev
from metagpt.logs import logger
from metagpt.schema import CodingContext, Document, RunCodeResult
from metagpt.utils.common import CodeParser
<<<<<<< HEAD
from metagpt.utils.file_repository import FileRepository
=======
>>>>>>> send18/dev
PROMPT_TEMPLATE = """
NOTICE
@ -89,12 +98,21 @@ class WriteCode(Action):
def __init__(self, name="WriteCode", context=None, llm=None):
super().__init__(name, context, llm)
<<<<<<< HEAD
@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(6))
async def write_code(self, prompt) -> str:
=======
def _is_invalid(self, filename):
return any(i in filename for i in ["mp3", "wav"])
@retry(stop=stop_after_attempt(2), wait=wait_fixed(1))
async def write_code(self, prompt):
>>>>>>> send18/dev
code_rsp = await self._aask(prompt)
code = CodeParser.parse_code(block="", text=code_rsp)
return code
<<<<<<< HEAD
async def run(self, *args, **kwargs) -> CodingContext:
bug_feedback = await FileRepository.get_file(filename=BUGFIX_FILENAME, relative_path=DOCS_FILE_REPO)
coding_context = CodingContext.loads(self.context.content)
@ -121,6 +139,11 @@ class WriteCode(Action):
summary_log=summary_doc.content if summary_doc else "",
)
logger.info(f"Writing {coding_context.filename}..")
=======
async def run(self, context, filename):
prompt = PROMPT_TEMPLATE.format(context=context, filename=filename)
logger.info(f"Writing {filename}..")
>>>>>>> send18/dev
code = await self.write_code(prompt)
if not coding_context.code_doc:
coding_context.code_doc = Document(filename=coding_context.filename, root_path=CONFIG.src_workspace)

43
metagpt/actions/write_prd.py
View file

@ -16,10 +16,13 @@ import json
from pathlib import Path
from typing import List
import aiofiles
from metagpt.actions import Action, ActionOutput
from metagpt.actions.fix_bug import FixBug
from metagpt.actions.search_and_summarize import SearchAndSummarize
from metagpt.config import CONFIG
<<<<<<< HEAD
from metagpt.const import (
COMPETITIVE_ANALYSIS_FILE_REPO,
DOCS_FILE_REPO,
@ -52,6 +55,11 @@ Requirements: According to the context, fill in the following missing informatio
ATTENTION: Output carefully referenced "Format example" in format.
## YOU NEED TO FULFILL THE BELOW JSON DOC
=======
from metagpt.logs import logger
from metagpt.utils.common import CodeParser
from metagpt.utils.mermaid import mermaid_to_file
>>>>>>> send18/dev
{{
"Language": "", # str, use the same language as the user requirement. en_us / zh_cn etc.
@ -237,7 +245,11 @@ OUTPUT_MAPPING = {
"Competitive Analysis": (List[str], ...),
"Competitive Quadrant Chart": (str, ...),
"Requirement Analysis": (str, ...),
<<<<<<< HEAD
"Requirement Pool": (List[List[str]], ...),
=======
"Requirement Pool": (List[Tuple[str, str]], ...),
>>>>>>> send18/dev
"UI Design draft": (str, ...),
"Anything UNCLEAR": (str, ...),
}
@ -376,6 +388,7 @@ class WritePRD(Action):
logger.info(sas.result)
logger.info(rsp)
<<<<<<< HEAD
# logger.info(format)
prompt_template, format_example = get_template(templates, format)
project_name = CONFIG.project_name if CONFIG.project_name else ""
@ -467,3 +480,33 @@ class WritePRD(Action):
if "YES" in res:
return True
return False
=======
prompt = PROMPT_TEMPLATE.format(
requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE
)
logger.debug(prompt)
prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING)
await self._save(prd.content)
return prd
async def _save_prd(self, docs_path, resources_path, prd):
prd_file = docs_path / "prd.md"
quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd)
await mermaid_to_file(
mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis"
)
async with aiofiles.open(prd_file, "w") as f:
await f.write(prd)
logger.info(f"Saving PRD to {prd_file}")
async def _save(self, prd):
workspace = CONFIG.workspace
workspace.mkdir(parents=True, exist_ok=True)
docs_path = workspace / "docs"
resources_path = workspace / "resources"
docs_path.mkdir(parents=True, exist_ok=True)
resources_path.mkdir(parents=True, exist_ok=True)
await self._save_prd(docs_path, resources_path, prd)
>>>>>>> send18/dev

159
metagpt/actions/write_teaching_plan.py Normal file
View file

@ -0,0 +1,159 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/7/27
@Author : mashenquan
@File : write_teaching_plan.py
"""
from metagpt.logs import logger
from metagpt.actions import Action
from metagpt.schema import Message
class TeachingPlanRequirement(Action):
"""Teaching Plan Requirement without any implementation details"""
async def run(self, *args, **kwargs):
raise NotImplementedError
class WriteTeachingPlanPart(Action):
"""Write Teaching Plan Part"""
def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"):
"""
:param name: action name
:param context: context
:param llm: object of :class:`LLM`
:param topic: topic part of teaching plan
:param language: A human language, such as Chinese, English, French, etc.
"""
super().__init__(name, context, llm)
self.topic = topic
self.language = language
self.rsp = None
async def run(self, messages, *args, **kwargs):
if len(messages) < 1 or not isinstance(messages[0], Message):
raise ValueError("Invalid args, a tuple of List[Message] is expected")
statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, [])
statements = []
from metagpt.roles import Role
for p in statement_patterns:
s = Role.format_value(p)
statements.append(s)
formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE
prompt = formatter.format(formation=self.FORMATION,
role=self.prefix,
statements="\n".join(statements),
lesson=messages[0].content,
topic=self.topic,
language=self.language)
logger.debug(prompt)
rsp = await self._aask(prompt=prompt)
logger.debug(rsp)
self._set_result(rsp)
return self.rsp
def _set_result(self, rsp):
if self.DATA_BEGIN_TAG in rsp:
ix = rsp.index(self.DATA_BEGIN_TAG)
rsp = rsp[ix + len(self.DATA_BEGIN_TAG):]
if self.DATA_END_TAG in rsp:
ix = rsp.index(self.DATA_END_TAG)
rsp = rsp[0:ix]
self.rsp = rsp.strip()
if self.topic != self.COURSE_TITLE:
return
if '#' not in self.rsp or self.rsp.index('#') != 0:
self.rsp = "# " + self.rsp
def __str__(self):
"""Return `topic` value when str()"""
return self.topic
def __repr__(self):
"""Show `topic` value when debug"""
return self.topic
FORMATION = "\"Capacity and role\" defines the role you are currently playing;\n" \
"\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n" \
"\t\"Statement\" defines the work detail you need to complete at this stage;\n" \
"\t\"Answer options\" defines the format requirements for your responses;\n" \
"\t\"Constraint\" defines the conditions that your responses must comply with."
COURSE_TITLE = "Title"
TOPICS = [
COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content",
"Teaching Methods and Strategies", "Learning Activities",
"Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement",
"Vocabulary Cloze", "Choice Questions", "Grammar Questions", "Translation Questions"
]
TOPIC_STATEMENTS = {
COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, "
"without anything else."],
"Teaching Content": [
"Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar "
"structures that appear in the textbook, as well as the listening materials and key points.",
"Statement: \"Teaching Content\" must include more examples."],
"Teaching Time Allocation": [
"Statement: \"Teaching Time Allocation\" must include how much time is allocated to each "
"part of the textbook content."],
"Teaching Methods and Strategies": [
"Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, "
"procedures, in detail."
],
"Vocabulary Cloze": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
"create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} "
"answers, and it should also include 10 {teaching_language} questions with {language} answers. "
"The key-related vocabulary and phrases in the textbook content must all be included in the exercises.",
],
"Grammar Questions": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
"create grammar questions. 10 questions."],
"Choice Questions": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
"create choice questions. 10 questions."],
"Translation Questions": [
"Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", "
"create translation questions. The translation should include 10 {language} questions with "
"{teaching_language} answers, and it should also include 10 {teaching_language} questions with "
"{language} answers."
]
}
# Teaching plan title
PROMPT_TITLE_TEMPLATE = "Do not refer to the context of the previous conversation records, " \
"start the conversation anew.\n\n" \
"Formation: {formation}\n\n" \
"{statements}\n" \
"Constraint: Writing in {language}.\n" \
"Answer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" " \
"and \"[TEACHING_PLAN_END]\" tags.\n" \
"[LESSON_BEGIN]\n" \
"{lesson}\n" \
"[LESSON_END]"
# Teaching plan parts:
PROMPT_TEMPLATE = "Do not refer to the context of the previous conversation records, " \
"start the conversation anew.\n\n" \
"Formation: {formation}\n\n" \
"Capacity and role: {role}\n" \
"Statement: Write the \"{topic}\" part of teaching plan, " \
"WITHOUT ANY content unrelated to \"{topic}\"!!\n" \
"{statements}\n" \
"Answer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" " \
"and \"[TEACHING_PLAN_END]\" tags.\n" \
"Answer options: Using proper markdown format from second-level header format.\n" \
"Constraint: Writing in {language}.\n" \
"[LESSON_BEGIN]\n" \
"{lesson}\n" \
"[LESSON_END]"
DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]"
DATA_END_TAG = "[TEACHING_PLAN_END]"

15
metagpt/config.py
View file

@ -7,17 +7,17 @@ Provide configuration, singleton
2. Add the parameter `src_workspace` for the old version project path.
"""
import datetime
import json
import os
from copy import deepcopy
from pathlib import Path
from typing import Any
from uuid import uuid4
import yaml
from metagpt.const import DEFAULT_WORKSPACE_ROOT, METAGPT_ROOT, OPTIONS
from metagpt.logs import logger
from metagpt.tools import SearchEngineType, WebBrowserEngineType
from metagpt.utils.cost_manager import CostManager
from metagpt.utils.singleton import Singleton
@ -46,13 +46,14 @@ class Config(metaclass=Singleton):
key_yaml_file = METAGPT_ROOT / "config/key.yaml"
default_yaml_file = METAGPT_ROOT / "config/config.yaml"
def __init__(self, yaml_file=default_yaml_file):
def __init__(self, yaml_file=default_yaml_file, cost_data=""):
self._init_with_config_files_and_env(yaml_file)
# The agent needs to be billed per user, so billing information cannot be destroyed when the session ends.
self.cost_manager = CostManager(**json.loads(cost_data)) if cost_data else CostManager()
logger.info("Config loading done.")
self._update()
def _update(self):
# logger.info("Config loading done.")
self.global_proxy = self._get("GLOBAL_PROXY")
self.openai_api_key = self._get("OPENAI_API_KEY")
self.anthropic_api_key = self._get("Anthropic_API_KEY")
@ -96,8 +97,7 @@ class Config(metaclass=Singleton):
self.long_term_memory = self._get("LONG_TERM_MEMORY", False)
if self.long_term_memory:
logger.warning("LONG_TERM_MEMORY is True")
self.max_budget = self._get("MAX_BUDGET", 10.0)
self.total_cost = 0.0
self.cost_manager.max_budget = self._get("MAX_BUDGET", 10.0)
self.code_review_k_times = 2
self.puppeteer_config = self._get("PUPPETEER_CONFIG", "")
@ -145,7 +145,8 @@ class Config(metaclass=Singleton):
return m.get(*args, **kwargs)
def get(self, key, *args, **kwargs):
"""Search for a value in config/key.yaml, config/config.yaml, and env; raise an error if not found"""
"""Retrieve values from config/key.yaml, config/config.yaml, and environment variables.
Throw an error if not found."""
value = self._get(key, *args, **kwargs)
if value is None:
raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file")

19
metagpt/const.py
View file

@ -12,9 +12,7 @@
import contextvars
import os
from pathlib import Path
from loguru import logger
import metagpt
OPTIONS = contextvars.ContextVar("OPTIONS")
@ -42,7 +40,7 @@ def get_metagpt_root():
# METAGPT PROJECT ROOT AND VARS
METAGPT_ROOT = get_metagpt_root()
METAGPT_ROOT = get_metagpt_root() # Dependent on METAGPT_PROJECT_ROOT
DEFAULT_WORKSPACE_ROOT = METAGPT_ROOT / "workspace"
DATA_PATH = METAGPT_ROOT / "data"
@ -93,3 +91,18 @@ CODE_SUMMARIES_FILE_REPO = "docs/code_summaries"
CODE_SUMMARIES_PDF_FILE_REPO = "resources/code_summaries"
YAPI_URL = "http://yapi.deepwisdomai.com/"
DEFAULT_LANGUAGE = "English"
DEFAULT_MAX_TOKENS = 1500
COMMAND_TOKENS = 500
BRAIN_MEMORY = "BRAIN_MEMORY"
SKILL_PATH = "SKILL_PATH"
SERPER_API_KEY = "SERPER_API_KEY"
# format
BASE64_FORMAT = "base64"
# REDIS
REDIS_KEY = "REDIS_KEY"

16
metagpt/document_store/faiss_store.py
View file

@ -21,10 +21,18 @@ from metagpt.logs import logger
class FaissStore(LocalStore):
<<<<<<< HEAD
def __init__(self, raw_data_path: Path, cache_dir=None, meta_col="source", content_col="output"):
self.meta_col = meta_col
self.content_col = content_col
super().__init__(raw_data_path, cache_dir)
=======
def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding_conf=None):
self.meta_col = meta_col
self.content_col = content_col
self.embedding_conf = embedding_conf or {}
super().__init__(raw_data, cache_dir)
>>>>>>> send18/dev
def _load(self) -> Optional["FaissStore"]:
index_file, store_file = self._get_index_and_store_fname()
@ -38,7 +46,7 @@ class FaissStore(LocalStore):
return store
def _write(self, docs, metadatas):
store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas)
store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07", **self.embedding_conf), metadatas=metadatas)
return store
def persist(self):
@ -84,6 +92,12 @@ class FaissStore(LocalStore):
if __name__ == "__main__":
faiss_store = FaissStore(DATA_PATH / "qcs/qcs_4w.json")
<<<<<<< HEAD
logger.info(faiss_store.search("Oily Skin Facial Cleanser"))
faiss_store.add([f"Oily Skin Facial Cleanser-{i}" for i in range(3)])
logger.info(faiss_store.search("Oily Skin Facial Cleanser"))
=======
logger.info(faiss_store.search("油皮洗面奶"))
faiss_store.add([f"油皮洗面奶-{i}" for i in range(3)])
logger.info(faiss_store.search("油皮洗面奶"))
>>>>>>> send18/dev

6
metagpt/learn/__init__.py
View file

@ -5,3 +5,9 @@
@Author : alexanderwu
@File : __init__.py
"""
from metagpt.learn.text_to_image import text_to_image
from metagpt.learn.text_to_speech import text_to_speech
from metagpt.learn.google_search import google_search
__all__ = ["text_to_image", "text_to_speech", "google_search"]

12
metagpt/learn/google_search.py Normal file
View file

@ -0,0 +1,12 @@
from metagpt.tools.search_engine import SearchEngine
async def google_search(query: str, max_results: int = 6, **kwargs):
"""Perform a web search and retrieve search results.
:param query: The search query.
:param max_results: The number of search results to retrieve
:return: The web search results in markdown format.
"""
resluts = await SearchEngine().run(query, max_results=max_results, as_string=False)
return "\n".join(f"{i}. [{j['title']}]({j['link']}): {j['snippet']}" for i, j in enumerate(resluts, 1))

123
metagpt/learn/skill_loader.py Normal file
View file

@ -0,0 +1,123 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/18
@Author : mashenquan
@File : skill_loader.py
@Desc : Skill YAML Configuration Loader.
"""
from pathlib import Path
from typing import Dict, List, Optional
import yaml
from pydantic import BaseModel, Field
from metagpt.config import CONFIG
class Example(BaseModel):
ask: str
answer: str
class Returns(BaseModel):
type: str
format: Optional[str] = None
class Parameter(BaseModel):
type: str
description: str = None
class Skill(BaseModel):
name: str
description: str = None
id: str = None
x_prerequisite: Dict = Field(default=None, alias="x-prerequisite")
parameters: Dict[str, Parameter] = None
examples: List[Example]
returns: Returns
@property
def arguments(self) -> Dict:
if not self.parameters:
return {}
ret = {}
for k, v in self.parameters.items():
ret[k] = v.description if v.description else ""
return ret
class Entity(BaseModel):
name: str = None
skills: List[Skill]
class Components(BaseModel):
pass
class SkillsDeclaration(BaseModel):
skillapi: str
entities: Dict[str, Entity]
components: Components = None
class SkillLoader:
def __init__(self, skill_yaml_file_name: Path = None):
if not skill_yaml_file_name:
skill_yaml_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml"
with open(str(skill_yaml_file_name), "r") as file:
skills = yaml.safe_load(file)
self._skills = SkillsDeclaration(**skills)
def get_skill_list(self, entity_name: str = "Assistant") -> Dict:
"""Return the skill name based on the skill description."""
entity = self.get_entity(entity_name)
if not entity:
return {}
agent_skills = CONFIG.agent_skills
if not agent_skills:
return {}
class AgentSkill(BaseModel):
name: str
names = [AgentSkill(**i).name for i in agent_skills]
description_to_name_mappings = {}
for s in entity.skills:
if s.name not in names:
continue
description_to_name_mappings[s.description] = s.name
return description_to_name_mappings
def get_skill(self, name, entity_name: str = "Assistant") -> Skill:
"""Return a skill by name."""
entity = self.get_entity(entity_name)
if not entity:
return None
for sk in entity.skills:
if sk.name == name:
return sk
def get_entity(self, name) -> Entity:
"""Return a list of skills for the entity."""
if not self._skills:
return None
return self._skills.entities.get(name)
if __name__ == "__main__":
CONFIG.agent_skills = [
{"id": 1, "name": "text_to_speech", "type": "builtin", "config": {}, "enabled": True},
{"id": 2, "name": "text_to_image", "type": "builtin", "config": {}, "enabled": True},
{"id": 3, "name": "ai_call", "type": "builtin", "config": {}, "enabled": True},
{"id": 3, "name": "data_analysis", "type": "builtin", "config": {}, "enabled": True},
{"id": 5, "name": "crawler", "type": "builtin", "config": {"engine": "ddg"}, "enabled": True},
{"id": 6, "name": "knowledge", "type": "builtin", "config": {}, "enabled": True},
]
loader = SkillLoader()
print(loader.get_skill_list())

24
metagpt/learn/text_to_embedding.py Normal file
View file

@ -0,0 +1,24 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/18
@Author : mashenquan
@File : text_to_embedding.py
@Desc : Text-to-Embedding skill, which provides text-to-embedding functionality.
"""
from metagpt.config import CONFIG
from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding
async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs):
"""Text to embedding
:param text: The text used for embedding.
:param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`.
:param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
:return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`.
"""
if CONFIG.OPENAI_API_KEY or openai_api_key:
return await oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key)
raise EnvironmentError

39
metagpt/learn/text_to_image.py Normal file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/18
@Author : mashenquan
@File : text_to_image.py
@Desc : Text-to-Image skill, which provides text-to-image functionality.
"""
import openai.error
from metagpt.config import CONFIG
from metagpt.const import BASE64_FORMAT
from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image
from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image
from metagpt.utils.s3 import S3
async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs):
"""Text to image
:param text: The text used for image conversion.
:param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
:param size_type: If using OPENAI, the available size options are ['256x256', '512x512', '1024x1024'], while for MetaGPT, the options are ['512x512', '512x768'].
:param model_url: MetaGPT model url
:return: The image data is returned in Base64 encoding.
"""
image_declaration = "data:image/png;base64,"
if CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL or model_url:
base64_data = await oas3_metagpt_text_to_image(text, size_type, model_url)
elif CONFIG.OPENAI_API_KEY or openai_api_key:
base64_data = await oas3_openai_text_to_image(text, size_type, openai_api_key)
else:
raise openai.error.InvalidRequestError("缺少必要的参数")
s3 = S3()
url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT)
if url:
return f"![{text}]({url})"
return image_declaration + base64_data if base64_data else ""

72
metagpt/learn/text_to_speech.py Normal file
View file

@ -0,0 +1,72 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/17
@Author : mashenquan
@File : text_to_speech.py
@Desc : Text-to-Speech skill, which provides text-to-speech functionality
"""
import openai
from metagpt.config import CONFIG
from metagpt.const import BASE64_FORMAT
from metagpt.tools.azure_tts import oas3_azsure_tts
from metagpt.tools.iflytek_tts import oas3_iflytek_tts
from metagpt.utils.s3 import S3
async def text_to_speech(
text,
lang="zh-CN",
voice="zh-CN-XiaomoNeural",
style="affectionate",
role="Girl",
subscription_key="",
region="",
iflytek_app_id="",
iflytek_api_key="",
iflytek_api_secret="",
**kwargs,
):
"""Text to speech
For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery`
:param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param text: The text used for voice conversion.
:param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint`
:param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API.
:param iflytek_app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`
:param iflytek_api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
:param iflytek_api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
:return: Returns the Base64-encoded .wav/.mp3 file data if successful, otherwise an empty string.
"""
if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region):
audio_declaration = "data:audio/wav;base64,"
base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region)
s3 = S3()
url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT)
if url:
return f"[{text}]({url})"
return audio_declaration + base64_data if base64_data else base64_data
if (CONFIG.IFLYTEK_APP_ID and CONFIG.IFLYTEK_API_KEY and CONFIG.IFLYTEK_API_SECRET) or (
iflytek_app_id and iflytek_api_key and iflytek_api_secret
):
audio_declaration = "data:audio/mp3;base64,"
base64_data = await oas3_iflytek_tts(
text=text, app_id=iflytek_app_id, api_key=iflytek_api_key, api_secret=iflytek_api_secret
)
s3 = S3()
url = await s3.cache(data=base64_data, file_ext=".mp3", format=BASE64_FORMAT)
if url:
return f"[{text}]({url})"
return audio_declaration + base64_data if base64_data else base64_data
raise openai.error.InvalidRequestError(
message="AZURE_TTS_SUBSCRIPTION_KEY, AZURE_TTS_REGION, IFLYTEK_APP_ID, IFLYTEK_API_KEY, IFLYTEK_API_SECRET error",
param={},
)

61
metagpt/llm.py
View file

@ -4,30 +4,63 @@
@Time : 2023/5/11 14:45
@Author : alexanderwu
@File : llm.py
@Modified By: mashenquan, 2023
"""
from enum import Enum
from metagpt.config import CONFIG
from metagpt.provider.anthropic_api import Claude2 as Claude
from metagpt.provider.openai_api import OpenAIGPTAPI
from metagpt.provider.zhipuai_api import ZhiPuAIGPTAPI
from metagpt.provider.spark_api import SparkAPI
from metagpt.provider.human_provider import HumanProvider
from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI
_ = HumanProvider() # Avoid pre-commit error
class LLMType(Enum):
OPENAI = "OpenAI"
METAGPT = "MetaGPT"
CLAUDE = "Claude"
UNKNOWN = "UNKNOWN"
@classmethod
def get(cls, value):
for member in cls:
if member.value == value:
return member
return cls.UNKNOWN
@classmethod
def __missing__(cls, value):
return cls.UNKNOWN
# Used in agents
class LLMFactory:
@staticmethod
def new_llm() -> "BaseGPTAPI":
# Determine which type of LLM to use based on the validity of the key.
if CONFIG.claude_api_key:
return Claude()
elif CONFIG.spark_api_key:
return SparkAPI()
elif CONFIG.zhipuai_api_key:
return ZhiPuAIGPTAPI()
# MetaGPT uses the same parameters as OpenAI.
constructors = {
LLMType.OPENAI.value: OpenAIGPTAPI,
LLMType.METAGPT.value: MetaGPTLLMAPI,
}
constructor = constructors.get(CONFIG.LLM_TYPE)
if constructor:
return constructor()
raise ValueError(f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}")
# Used in metagpt
def LLM() -> "BaseGPTAPI":
""" initialize different LLM instance according to the key field existence"""
# TODO a little trick, can use registry to initialize LLM instance further
if CONFIG.openai_api_key:
llm = OpenAIGPTAPI()
elif CONFIG.claude_api_key:
llm = Claude()
elif CONFIG.spark_api_key:
llm = SparkAPI()
elif CONFIG.zhipuai_api_key:
llm = ZhiPuAIGPTAPI()
else:
raise RuntimeError("You should config a LLM configuration first")
return llm
return LLMFactory.new_llm()

7
metagpt/management/skill_manager.py
View file

@ -4,11 +4,11 @@
@Time : 2023/6/5 01:44
@Author : alexanderwu
@File : skill_manager.py
@Modified By: mashenquan, 2023/8/20. Remove useless `_llm`
"""
from metagpt.actions import Action
from metagpt.const import PROMPT_PATH
from metagpt.document_store.chromadb_store import ChromaStore
from metagpt.llm import LLM
from metagpt.logs import logger
Skill = Action
@ -18,9 +18,14 @@ class SkillManager:
"""Used to manage all skills"""
def __init__(self):
<<<<<<< HEAD
self._llm = LLM()
self._store = ChromaStore("skill_manager")
self._skills: dict[str:Skill] = {}
=======
self._store = ChromaStore('skill_manager')
self._skills: dict[str: Skill] = {}
>>>>>>> send18/dev
def add_skill(self, skill: Skill):
"""

348
metagpt/memory/brain_memory.py Normal file
View file

@ -0,0 +1,348 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/18
@Author : mashenquan
@File : brain_memory.py
@Desc : Support memory for multiple tasks and multiple mainlines.
@Modified By: mashenquan, 2023/9/4. + redis memory cache.
"""
import json
import re
from enum import Enum
from typing import Dict, List, Optional
import openai
import pydantic
from metagpt import Message
from metagpt.config import CONFIG
from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS
from metagpt.llm import LLMType
from metagpt.logs import logger
from metagpt.schema import RawMessage
from metagpt.utils.redis import Redis
class MessageType(Enum):
Talk = "TALK"
Solution = "SOLUTION"
Problem = "PROBLEM"
Skill = "SKILL"
Answer = "ANSWER"
class BrainMemory(pydantic.BaseModel):
history: List[Dict] = []
stack: List[Dict] = []
solution: List[Dict] = []
knowledge: List[Dict] = []
historical_summary: str = ""
last_history_id: str = ""
is_dirty: bool = False
last_talk: str = None
llm_type: Optional[str] = None
cacheable: bool = True
def add_talk(self, msg: Message):
msg.add_tag(MessageType.Talk.value)
self.add_history(msg)
self.is_dirty = True
def add_answer(self, msg: Message):
msg.add_tag(MessageType.Answer.value)
self.add_history(msg)
self.is_dirty = True
def get_knowledge(self) -> str:
texts = [Message(**m).content for m in self.knowledge]
return "\n".join(texts)
@staticmethod
async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory":
redis = Redis(conf=redis_conf)
if not redis.is_valid() or not redis_key:
return BrainMemory(llm_type=CONFIG.LLM_TYPE)
v = await redis.get(key=redis_key)
logger.debug(f"REDIS GET {redis_key} {v}")
if v:
data = json.loads(v)
bm = BrainMemory(**data)
bm.is_dirty = False
return bm
return BrainMemory(llm_type=CONFIG.LLM_TYPE)
async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None):
if not self.is_dirty:
return
redis = Redis(conf=redis_conf)
if not redis.is_valid() or not redis_key:
return False
v = self.json()
if self.cacheable:
await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec)
logger.debug(f"REDIS SET {redis_key} {v}")
self.is_dirty = False
@staticmethod
def to_redis_key(prefix: str, user_id: str, chat_id: str):
return f"{prefix}:{user_id}:{chat_id}"
async def set_history_summary(self, history_summary, redis_key, redis_conf):
if self.historical_summary == history_summary:
if self.is_dirty:
await self.dumps(redis_key=redis_key, redis_conf=redis_conf)
self.is_dirty = False
return
self.historical_summary = history_summary
self.history = []
await self.dumps(redis_key=redis_key, redis_conf=redis_conf)
self.is_dirty = False
def add_history(self, msg: Message):
if msg.id:
if self.to_int(msg.id, 0) <= self.to_int(self.last_history_id, -1):
return
self.history.append(msg.dict())
self.last_history_id = str(msg.id)
self.is_dirty = True
def exists(self, text) -> bool:
for m in reversed(self.history):
if m.get("content") == text:
return True
return False
@staticmethod
def to_int(v, default_value):
try:
return int(v)
except:
return default_value
def pop_last_talk(self):
v = self.last_talk
self.last_talk = None
return v
async def summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs):
if self.llm_type == LLMType.METAGPT.value:
return await self._metagpt_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs)
return await self._openai_summarize(
llm=llm, max_words=max_words, keep_language=keep_language, limit=limit, **kwargs
)
async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs):
max_token_count = DEFAULT_MAX_TOKENS
max_count = 100
texts = [self.historical_summary]
for i in self.history:
m = Message(**i)
texts.append(m.content)
text = "\n".join(texts)
text_length = len(text)
if limit > 0 and text_length < limit:
return text
summary = ""
while max_count > 0:
if text_length < max_token_count:
summary = await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language)
break
padding_size = 20 if max_token_count > 20 else 0
text_windows = self.split_texts(text, window_size=max_token_count - padding_size)
part_max_words = min(int(max_words / len(text_windows)) + 1, 100)
summaries = []
for ws in text_windows:
response = await self._get_summary(
text=ws, llm=llm, max_words=part_max_words, keep_language=keep_language
)
summaries.append(response)
if len(summaries) == 1:
summary = summaries[0]
break
# Merged and retry
text = "\n".join(summaries)
text_length = len(text)
max_count -= 1 # safeguard
if summary:
await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS)
return summary
raise openai.error.InvalidRequestError(message="text too long", param=None)
async def _metagpt_summarize(self, max_words=200, **kwargs):
if not self.history:
return ""
total_length = 0
msgs = []
for i in reversed(self.history):
m = Message(**i)
delta = len(m.content)
if total_length + delta > max_words:
left = max_words - total_length
if left == 0:
break
m.content = m.content[0:left]
msgs.append(m.dict())
break
msgs.append(i)
total_length += delta
msgs.reverse()
self.history = msgs
self.is_dirty = True
await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF)
self.is_dirty = False
return BrainMemory.to_metagpt_history_format(self.history)
@staticmethod
def to_metagpt_history_format(history) -> str:
mmsg = []
for m in history:
msg = Message(**m)
r = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content)
mmsg.append(r)
return json.dumps(mmsg)
@staticmethod
async def _get_summary(text: str, llm, max_words=20, keep_language: bool = False):
"""Generate text summary"""
if len(text) < max_words:
return text
if keep_language:
command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly."
else:
command = f"Translate the above content into a summary of less than {max_words} words."
msg = text + "\n\n" + command
logger.debug(f"summary ask:{msg}")
response = await llm.aask(msg=msg, system_msgs=[])
logger.debug(f"summary rsp: {response}")
return response
async def get_title(self, llm, max_words=5, **kwargs) -> str:
"""Generate text title"""
if self.llm_type == LLMType.METAGPT.value:
return Message(**self.history[0]).content if self.history else "New"
summary = await self.summarize(llm=llm, max_words=500)
language = CONFIG.language or DEFAULT_LANGUAGE
command = f"Translate the above summary into a {language} title of less than {max_words} words."
summaries = [summary, command]
msg = "\n".join(summaries)
logger.debug(f"title ask:{msg}")
response = await llm.aask(msg=msg, system_msgs=[])
logger.debug(f"title rsp: {response}")
return response
async def is_related(self, text1, text2, llm):
if self.llm_type == LLMType.METAGPT.value:
return await self._metagpt_is_related(text1=text1, text2=text2, llm=llm)
return await self._openai_is_related(text1=text1, text2=text2, llm=llm)
@staticmethod
async def _metagpt_is_related(**kwargs):
return False
@staticmethod
async def _openai_is_related(text1, text2, llm, **kwargs):
# command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]."
command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear."
rsp = await llm.aask(msg=command, system_msgs=[])
result = True if "TRUE" in rsp else False
p2 = text2.replace("\n", "")
p1 = text1.replace("\n", "")
logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n")
return result
async def rewrite(self, sentence: str, context: str, llm):
if self.llm_type == LLMType.METAGPT.value:
return await self._metagpt_rewrite(sentence=sentence, context=context, llm=llm)
return await self._openai_rewrite(sentence=sentence, context=context, llm=llm)
async def _metagpt_rewrite(self, sentence: str, **kwargs):
return sentence
async def _openai_rewrite(self, sentence: str, context: str, llm, **kwargs):
# command = (
# f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}"
# )
command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}"
rsp = await llm.aask(msg=command, system_msgs=[])
logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n")
return rsp
@staticmethod
def split_texts(text: str, window_size) -> List[str]:
"""Splitting long text into sliding windows text"""
if window_size <= 0:
window_size = BrainMemory.DEFAULT_TOKEN_SIZE
total_len = len(text)
if total_len <= window_size:
return [text]
padding_size = 20 if window_size > 20 else 0
windows = []
idx = 0
data_len = window_size - padding_size
while idx < total_len:
if window_size + idx > total_len: # 不足一个滑窗
windows.append(text[idx:])
break
# 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....]
# window_size=3, padding_size=1：
# [1, 2, 3], [3, 4, 5], [5, 6, 7], ....
# idx=2, | idx=5 | idx=8 | ...
w = text[idx : idx + window_size]
windows.append(w)
idx += data_len
return windows
@staticmethod
def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"):
match = re.match(pattern, input_string)
if match:
return match.group(1), match.group(2)
else:
return None, input_string
def set_llm_type(self, v):
if v and v != self.llm_type:
self.llm_type = v
self.is_dirty = True
@property
def is_history_available(self):
return bool(self.history or self.historical_summary)
@property
def history_text(self):
if self.llm_type == LLMType.METAGPT.value:
return self._get_metagpt_history_text()
return self._get_openai_history_text()
def _get_metagpt_history_text(self):
return BrainMemory.to_metagpt_history_format(self.history)
def _get_openai_history_text(self):
if len(self.history) == 0 and not self.historical_summary:
return ""
texts = [self.historical_summary] if self.historical_summary else []
for m in self.history[:-1]:
if isinstance(m, Dict):
t = Message(**m).content
elif isinstance(m, Message):
t = m.content
else:
continue
texts.append(t)
return "\n".join(texts)
DEFAULT_TOKEN_SIZE = 500

1
metagpt/memory/longterm_memory.py
View file

@ -2,6 +2,7 @@
# -*- coding: utf-8 -*-
"""
@Desc : the implement of Long-term memory
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
from metagpt.logs import logger

8
metagpt/memory/memory.py
View file

@ -88,3 +88,11 @@ class Memory:
continue
rsp += self.index[action]
return rsp
def get_by_tags(self, tags: list) -> list[Message]:
"""Return messages with specified tags"""
result = []
for m in self.storage:
if m.is_contain_tags(tags):
result.append(m)
return result

5
metagpt/memory/memory_storage.py
View file

@ -1,6 +1,9 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# @Desc : the implement of memory storage
"""
@Desc : the implement of memory storage
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
from pathlib import Path
from typing import List

4
metagpt/provider/__init__.py
View file

@ -4,9 +4,11 @@
@Time : 2023/5/5 22:59
@Author : alexanderwu
@File : __init__.py
@Modified By: mashenquan, 2023/9/8. Add `MetaGPTLLMAPI`
"""
from metagpt.provider.openai_api import OpenAIGPTAPI
from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI
__all__ = ["OpenAIGPTAPI"]
__all__ = ["OpenAIGPTAPI", "MetaGPTLLMAPI"]

1
metagpt/provider/base_gpt_api.py
View file

@ -4,6 +4,7 @@
@Time : 2023/5/5 23:04
@Author : alexanderwu
@File : base_gpt_api.py
@Desc : mashenquan, 2023/8/22. + try catch
"""
import json
from abc import abstractmethod

278
metagpt/provider/metagpt_llm_api.py Normal file
View file

@ -0,0 +1,278 @@
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/30
@Author : mashenquan
@File : metagpt_llm_api.py
@Desc : MetaGPT LLM related APIs
"""
from metagpt.provider.openai_api import OpenAIGPTAPI
# from metagpt.provider.base_gpt_api import BaseGPTAPI
# from metagpt.provider.openai_api import RateLimiter
class MetaGPTLLMAPI(OpenAIGPTAPI):
"""MetaGPT LLM api"""
def __init__(self):
super(MetaGPTLLMAPI, self).__init__()
# def __init__(self):
# self.__init_openai(CONFIG)
# self.llm = openai
# self.model = CONFIG.openai_api_model
# self.auto_max_tokens = False
# self._cost_manager = CostManager()
# RateLimiter.__init__(self, rpm=self.rpm)
#
# def __init_openai(self, config):
# openai.api_key = config.openai_api_key
# if config.openai_api_base:
# openai.api_base = config.openai_api_base
# if config.openai_api_type:
# openai.api_type = config.openai_api_type
# openai.api_version = config.openai_api_version
# self.rpm = int(config.get("RPM", 10))
#
# async def _achat_completion_stream(self, messages: list[dict]) -> str:
# response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True)
#
# # create variables to collect the stream of chunks
# collected_chunks = []
# collected_messages = []
# # iterate through the stream of events
# async for chunk in response:
# collected_chunks.append(chunk) # save the event response
# choices = chunk["choices"]
# if len(choices) > 0:
# chunk_message = chunk["choices"][0].get("delta", {}) # extract the message
# collected_messages.append(chunk_message) # save the message
# if "content" in chunk_message:
# print(chunk_message["content"], end="")
# print()
#
# full_reply_content = "".join([m.get("content", "") for m in collected_messages])
# usage = self._calc_usage(messages, full_reply_content)
# self._update_costs(usage)
# return full_reply_content
#
# def _cons_kwargs(self, messages: list[dict], **configs) -> dict:
# kwargs = {
# "messages": messages,
# "max_tokens": self.get_max_tokens(messages),
# "n": 1,
# "stop": None,
# "temperature": 0.3,
# "timeout": 3,
# }
# if configs:
# kwargs.update(configs)
#
# if CONFIG.openai_api_type == "azure":
# if CONFIG.deployment_name and CONFIG.deployment_id:
# raise ValueError("You can only use one of the `deployment_id` or `deployment_name` model")
# elif not CONFIG.deployment_name and not CONFIG.deployment_id:
# raise ValueError("You must specify `DEPLOYMENT_NAME` or `DEPLOYMENT_ID` parameter")
# kwargs_mode = (
# {"engine": CONFIG.deployment_name}
# if CONFIG.deployment_name
# else {"deployment_id": CONFIG.deployment_id}
# )
# else:
# kwargs_mode = {"model": self.model}
# kwargs.update(kwargs_mode)
# return kwargs
#
# async def _achat_completion(self, messages: list[dict]) -> dict:
# rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages))
# self._update_costs(rsp.get("usage"))
# return rsp
#
# def _chat_completion(self, messages: list[dict]) -> dict:
# rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages))
# self._update_costs(rsp)
# return rsp
#
# def completion(self, messages: list[dict]) -> dict:
# # if isinstance(messages[0], Message):
# # messages = self.messages_to_dict(messages)
# return self._chat_completion(messages)
#
# async def acompletion(self, messages: list[dict]) -> dict:
# # if isinstance(messages[0], Message):
# # messages = self.messages_to_dict(messages)
# return await self._achat_completion(messages)
#
# @retry(
# wait=wait_random_exponential(min=1, max=60),
# stop=stop_after_attempt(6),
# after=after_log(logger, logger.level("WARNING").name),
# retry=retry_if_exception_type(APIConnectionError),
# retry_error_callback=log_and_reraise,
# )
# async def acompletion_text(self, messages: list[dict], stream=False) -> str:
# """when streaming, print each token in place."""
# if stream:
# return await self._achat_completion_stream(messages)
# rsp = await self._achat_completion(messages)
# return self.get_choice_text(rsp)
#
# def _func_configs(self, messages: list[dict], **kwargs) -> dict:
# """
# Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
# """
# if "tools" not in kwargs:
# configs = {
# "tools": [{"type": "function", "function": GENERAL_FUNCTION_SCHEMA}],
# "tool_choice": GENERAL_TOOL_CHOICE,
# }
# kwargs.update(configs)
#
# return self._cons_kwargs(messages, **kwargs)
#
# def _chat_completion_function(self, messages: list[dict], **kwargs) -> dict:
# rsp = self.llm.ChatCompletion.create(**self._func_configs(messages, **kwargs))
# self._update_costs(rsp.get("usage"))
# return rsp
#
# async def _achat_completion_function(self, messages: list[dict], **chat_configs) -> dict:
# rsp = await self.llm.ChatCompletion.acreate(**self._func_configs(messages, **chat_configs))
# self._update_costs(rsp.get("usage"))
# return rsp
#
# def _process_message(self, messages: Union[str, Message, list[dict], list[Message], list[str]]) -> list[dict]:
# """convert messages to list[dict]."""
# if isinstance(messages, list):
# messages = [Message(msg) if isinstance(msg, str) else msg for msg in messages]
# return [msg if isinstance(msg, dict) else msg.to_dict() for msg in messages]
#
# if isinstance(messages, Message):
# messages = [messages.to_dict()]
# elif isinstance(messages, str):
# messages = [{"role": "user", "content": messages}]
# else:
# raise ValueError(
# f"Only support messages type are: str, Message, list[dict], but got {type(messages).__name__}!"
# )
# return messages
#
# def ask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict:
# """Use function of tools to ask a code.
#
# Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
#
# Examples:
#
# >>> llm = OpenAIGPTAPI()
# >>> llm.ask_code("Write a python hello world code.")
# {'language': 'python', 'code': "print('Hello, World!')"}
# >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}]
# >>> llm.ask_code(msg)
# {'language': 'python', 'code': "print('Hello, World!')"}
# """
# messages = self._process_message(messages)
# rsp = self._chat_completion_function(messages, **kwargs)
# return self.get_choice_function_arguments(rsp)
#
# async def aask_code(self, messages: Union[str, Message, list[dict]], **kwargs) -> dict:
# """Use function of tools to ask a code.
#
# Note: Keep kwargs consistent with the parameters in the https://platform.openai.com/docs/api-reference/chat/create
#
# Examples:
#
# >>> llm = OpenAIGPTAPI()
# >>> rsp = await llm.ask_code("Write a python hello world code.")
# >>> rsp
# {'language': 'python', 'code': "print('Hello, World!')"}
# >>> msg = [{'role': 'user', 'content': "Write a python hello world code."}]
# >>> rsp = await llm.aask_code(msg) # -> {'language': 'python', 'code': "print('Hello, World!')"}
# """
# messages = self._process_message(messages)
# rsp = await self._achat_completion_function(messages, **kwargs)
# return self.get_choice_function_arguments(rsp)
#
# def _calc_usage(self, messages: list[dict], rsp: str) -> dict:
# usage = {}
# if CONFIG.calc_usage:
# try:
# prompt_tokens = count_message_tokens(messages, self.model)
# completion_tokens = count_string_tokens(rsp, self.model)
# usage["prompt_tokens"] = prompt_tokens
# usage["completion_tokens"] = completion_tokens
# return usage
# except Exception as e:
# logger.error("usage calculation failed!", e)
# else:
# return usage
#
# async def acompletion_batch(self, batch: list[list[dict]]) -> list[dict]:
# """Return full JSON"""
# split_batches = self.split_batches(batch)
# all_results = []
#
# for small_batch in split_batches:
# logger.info(small_batch)
# await self.wait_if_needed(len(small_batch))
#
# future = [self.acompletion(prompt) for prompt in small_batch]
# results = await asyncio.gather(*future)
# logger.info(results)
# all_results.extend(results)
#
# return all_results
#
# async def acompletion_batch_text(self, batch: list[list[dict]]) -> list[str]:
# """Only return plain text"""
# raw_results = await self.acompletion_batch(batch)
# results = []
# for idx, raw_result in enumerate(raw_results, start=1):
# result = self.get_choice_text(raw_result)
# results.append(result)
# logger.info(f"Result of task {idx}: {result}")
# return results
#
# def _update_costs(self, usage: dict):
# if CONFIG.calc_usage:
# try:
# prompt_tokens = int(usage["prompt_tokens"])
# completion_tokens = int(usage["completion_tokens"])
# self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
# except Exception as e:
# logger.error("updating costs failed!", e)
#
# def get_costs(self) -> Costs:
# return self._cost_manager.get_costs()
#
# def get_max_tokens(self, messages: list[dict]):
# if not self.auto_max_tokens:
# return CONFIG.max_tokens_rsp
# return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp)
#
# def moderation(self, content: Union[str, list[str]]):
# try:
# if not content:
# logger.error("content cannot be empty!")
# else:
# rsp = self._moderation(content=content)
# return rsp
# except Exception as e:
# logger.error(f"moderating failed:{e}")
#
# def _moderation(self, content: Union[str, list[str]]):
# rsp = self.llm.Moderation.create(input=content)
# return rsp
#
# async def amoderation(self, content: Union[str, list[str]]):
# try:
# if not content:
# logger.error("content cannot be empty!")
# else:
# rsp = await self._amoderation(content=content)
# return rsp
# except Exception as e:
# logger.error(f"moderating failed:{e}")
#
# async def _amoderation(self, content: Union[str, list[str]]):
# rsp = await self.llm.Moderation.acreate(input=content)
# return rsp

101
metagpt/provider/openai_api.py
View file

@ -8,13 +8,13 @@
@Modified By: mashenquan, 2023/11/21. Fix bug: ReadTimeout.
@Modified By: mashenquan, 2023/12/1. Fix bug: Unclosed connection caused by openai 0.x.
"""
import asyncio
import time
from typing import NamedTuple, Union
import openai
from typing import Union
from openai import APIConnectionError, AsyncAzureOpenAI, AsyncOpenAI, RateLimitError
from openai.types import CompletionUsage
import asyncio
import time
import openai
from tenacity import (
after_log,
retry,
@ -22,15 +22,14 @@ from tenacity import (
stop_after_attempt,
wait_random_exponential,
)
from metagpt.config import CONFIG
from metagpt.llm import LLMType
from metagpt.logs import logger
from metagpt.provider.base_gpt_api import BaseGPTAPI
from metagpt.provider.constant import GENERAL_FUNCTION_SCHEMA, GENERAL_TOOL_CHOICE
from metagpt.schema import Message
from metagpt.utils.singleton import Singleton
from metagpt.utils.cost_manager import Costs
from metagpt.utils.token_counter import (
TOKEN_COSTS,
count_message_tokens,
count_string_tokens,
get_max_completion_tokens,
@ -62,75 +61,6 @@ class RateLimiter:
self.last_call_time = time.time()
class Costs(NamedTuple):
total_prompt_tokens: int
total_completion_tokens: int
total_cost: float
total_budget: float
class CostManager(metaclass=Singleton):
"""计算使用接口的开销"""
def __init__(self):
self.total_prompt_tokens = 0
self.total_completion_tokens = 0
self.total_cost = 0
self.total_budget = 0
def update_cost(self, prompt_tokens, completion_tokens, model):
"""
Update the total cost, prompt tokens, and completion tokens.
Args:
prompt_tokens (int): The number of tokens used in the prompt.
completion_tokens (int): The number of tokens used in the completion.
model (str): The model used for the API call.
"""
self.total_prompt_tokens += prompt_tokens
self.total_completion_tokens += completion_tokens
cost = (
prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]
) / 1000
self.total_cost += cost
logger.info(
f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | "
f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}"
)
CONFIG.total_cost = self.total_cost
def get_total_prompt_tokens(self):
"""
Get the total number of prompt tokens.
Returns:
int: The total number of prompt tokens.
"""
return self.total_prompt_tokens
def get_total_completion_tokens(self):
"""
Get the total number of completion tokens.
Returns:
int: The total number of completion tokens.
"""
return self.total_completion_tokens
def get_total_cost(self):
"""
Get the total cost of API calls.
Returns:
float: The total cost of API calls.
"""
return self.total_cost
def get_costs(self) -> Costs:
"""Get all costs"""
return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget)
def log_and_reraise(retry_state):
logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}")
logger.warning(
@ -161,7 +91,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
else:
# https://github.com/openai/openai-python#async-usage
self._client = AsyncOpenAI(api_key=CONFIG.openai_api_key, base_url=CONFIG.openai_api_base)
self._cost_manager = CostManager()
RateLimiter.__init__(self, rpm=self.rpm)
async def _achat_completion_stream(self, messages: list[dict], timeout=3) -> str:
@ -362,12 +291,15 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
def _update_costs(self, usage: CompletionUsage):
if CONFIG.calc_usage:
prompt_tokens = usage.prompt_tokens
completion_tokens = usage.completion_tokens
self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
try:
prompt_tokens = usage.prompt_tokens
completion_tokens = usage.completion_tokens
CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model)
except Exception as e:
logger.error("updating costs failed!", e)
def get_costs(self) -> Costs:
return self._cost_manager.get_costs()
return CONFIG.cost_manager.get_costs()
def get_max_tokens(self, messages: list[dict]):
if not self.auto_max_tokens:
@ -410,3 +342,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter):
return loop
else:
raise e
async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str:
from metagpt.memory.brain_memory import BrainMemory
memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False)
return await memory.summarize(llm=self, max_words=max_words, keep_language=keep_language)

159
metagpt/roles/assistant.py Normal file
View file

@ -0,0 +1,159 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/7
@Author : mashenquan
@File : assistant.py
@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the
ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to
make these symbols configurable and standardized, making the process of building flows more convenient.
For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html`
This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a
configuration file.
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false
indicates that further reasoning cannot continue.
"""
import asyncio
from pathlib import Path
from metagpt.actions import ActionOutput
from metagpt.actions.skill_action import ArgumentsParingAction, SkillAction
from metagpt.actions.talk_action import TalkAction
from metagpt.config import CONFIG
from metagpt.learn.skill_loader import SkillLoader
from metagpt.logs import logger
from metagpt.memory.brain_memory import BrainMemory, MessageType
from metagpt.roles import Role
from metagpt.schema import Message
class Assistant(Role):
"""Assistant for solving common issues."""
def __init__(
self,
name="Lily",
profile="An assistant",
goal="Help to solve problem",
constraints="Talk in {language}",
desc="",
*args,
**kwargs,
):
super(Assistant, self).__init__(
name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs
)
brain_memory = CONFIG.BRAIN_MEMORY
self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory(llm_type=CONFIG.LLM_TYPE)
skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None
self.skills = SkillLoader(skill_yaml_file_name=skill_path)
async def think(self) -> bool:
"""Everything will be done part by part."""
last_talk = await self.refine_memory()
if not last_talk:
return False
prompt = ""
skills = self.skills.get_skill_list()
for desc, name in skills.items():
prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n"
prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n'
prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n"
rsp = await self._llm.aask(prompt, [])
logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n")
return await self._plan(rsp, last_talk=last_talk)
async def act(self) -> ActionOutput:
result = await self._rc.todo.run(**CONFIG.options)
if not result:
return None
if isinstance(result, str):
msg = Message(content=result)
output = ActionOutput(content=result)
else:
msg = Message(
content=result.content, instruct_content=result.instruct_content, cause_by=type(self._rc.todo)
)
output = result
self.memory.add_answer(msg)
return output
async def talk(self, text):
self.memory.add_talk(Message(content=text))
async def _plan(self, rsp: str, **kwargs) -> bool:
skill, text = BrainMemory.extract_info(input_string=rsp)
handlers = {
MessageType.Talk.value: self.talk_handler,
MessageType.Skill.value: self.skill_handler,
}
handler = handlers.get(skill, self.talk_handler)
return await handler(text, **kwargs)
async def talk_handler(self, text, **kwargs) -> bool:
history = self.memory.history_text
text = kwargs.get("last_talk") or text
action = TalkAction(
talk=text, knowledge=self.memory.get_knowledge(), history_summary=history, llm=self._llm, **kwargs
)
self.add_to_do(action)
return True
async def skill_handler(self, text, **kwargs) -> bool:
last_talk = kwargs.get("last_talk")
skill = self.skills.get_skill(text)
if not skill:
logger.info(f"skill not found: {text}")
return await self.talk_handler(text=last_talk, **kwargs)
action = ArgumentsParingAction(skill=skill, llm=self._llm, **kwargs)
await action.run(**kwargs)
if action.args is None:
return await self.talk_handler(text=last_talk, **kwargs)
action = SkillAction(skill=skill, args=action.args, llm=self._llm, name=skill.name, desc=skill.description)
self.add_to_do(action)
return True
async def refine_memory(self) -> str:
last_talk = self.memory.pop_last_talk()
if last_talk is None: # No user feedback, unsure if past conversation is finished.
return None
if not self.memory.is_history_available:
return last_talk
history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm)
if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm):
# Merge relevant content.
last_talk = await self.memory.rewrite(sentence=last_talk, context=history_summary, llm=self._llm)
return last_talk
return last_talk
def get_memory(self) -> str:
return self.memory.json()
def load_memory(self, jsn):
try:
self.memory = BrainMemory(**jsn)
except Exception as e:
logger.exception(f"load error:{e}, data:{jsn}")
async def main():
topic = "what's apple"
role = Assistant(language="Chinese")
await role.talk(topic)
while True:
has_action = await role.think()
if not has_action:
break
msg = await role.act()
logger.info(msg)
# Retrieve user terminal input.
logger.info("Enter prompt")
talk = input("You: ")
await role.talk(talk)
if __name__ == "__main__":
CONFIG.language = "Chinese"
asyncio.run(main())

166
metagpt/roles/engineer.py
View file

@ -16,13 +16,19 @@
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
<<<<<<< HEAD
from __future__ import annotations
import json
from collections import defaultdict
=======
import asyncio
from collections import OrderedDict
>>>>>>> send18/dev
from pathlib import Path
from typing import Set
<<<<<<< HEAD
from metagpt.actions import Action, WriteCode, WriteCodeReview, WriteTasks
from metagpt.actions.fix_bug import FixBug
from metagpt.actions.summarize_code import SummarizeCode
@ -43,6 +49,18 @@ from metagpt.schema import (
Message,
)
from metagpt.utils.common import any_to_name, any_to_str, any_to_str_set
=======
import aiofiles
from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
from metagpt.utils.common import CodeParser
from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP
>>>>>>> send18/dev
IS_PASS_PROMPT = """
{context}
@ -67,6 +85,7 @@ class Engineer(Role):
use_code_review (bool): Whether to use code review.
"""
<<<<<<< HEAD
def __init__(
self,
name: str = "Alex",
@ -77,6 +96,18 @@ class Engineer(Role):
use_code_review: bool = False,
) -> None:
"""Initializes the Engineer role with given attributes."""
=======
class Engineer(Role):
def __init__(
self,
name="Alex",
profile="Engineer",
goal="Write elegant, readable, extensible, efficient code",
constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain",
n_borg=1,
use_code_review=False,
):
>>>>>>> send18/dev
super().__init__(name, profile, goal, constraints)
self.use_code_review = use_code_review
self._watch([WriteTasks, SummarizeCode, WriteCode, WriteCodeReview, FixBug])
@ -90,6 +121,7 @@ class Engineer(Role):
m = json.loads(task_msg.content)
return m.get("Task list")
<<<<<<< HEAD
async def _act_sp_with_cr(self, review=False) -> Set[str]:
changed_files = set()
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
@ -113,8 +145,83 @@ class Engineer(Role):
msg = Message(
content=coding_context.json(), instruct_content=coding_context, role=self.profile, cause_by=WriteCode
)
self._rc.memory.add(msg)
=======
@classmethod
def parse_tasks(self, task_msg: Message) -> list[str]:
if task_msg.instruct_content:
return task_msg.instruct_content.dict().get("Task list")
return CodeParser.parse_file_list(block="Task list", text=task_msg.content)
@classmethod
def parse_code(self, code_text: str) -> str:
return CodeParser.parse_code(block="", text=code_text)
@classmethod
def parse_workspace(cls, system_design_msg: Message) -> str:
if system_design_msg.instruct_content:
return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"')
return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
def get_workspace(self) -> Path:
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
if not msg:
return CONFIG.workspace / "src"
workspace = self.parse_workspace(msg)
# Codes are written in workspace/{package_name}/{package_name}
return CONFIG.workspace / workspace
async def write_file(self, filename: str, code: str):
workspace = self.get_workspace()
filename = filename.replace('"', "").replace("\n", "")
file = workspace / filename
file.parent.mkdir(parents=True, exist_ok=True)
async with aiofiles.open(file, "w") as f:
await f.write(code)
return file
def recv(self, message: Message) -> None:
self._rc.memory.add(message)
if message in self._rc.important_memory:
self.todos = self.parse_tasks(message)
async def _act_mp(self) -> Message:
# self.recreate_workspace()
todo_coros = []
for todo in self.todos:
todo_coro = WriteCode().run(
context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo
)
todo_coros.append(todo_coro)
rsps = await gather_ordered_k(todo_coros, self.n_borg)
for todo, code_rsp in zip(self.todos, rsps):
_ = self.parse_code(code_rsp)
logger.info(todo)
logger.info(code_rsp)
# self.write_file(todo, code)
msg = Message(content=code_rsp, role=self.profile, cause_by=type(self._rc.todo))
self._rc.memory.add(msg)
del self.todos[0]
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo))
return msg
async def _act_sp(self) -> Message:
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
instruct_content = {}
for todo in self.todos:
code = await WriteCode().run(context=self._rc.history, filename=todo)
# logger.info(todo)
# logger.info(code_rsp)
# code = self.parse_code(code_rsp)
file_path = await self.write_file(todo, code)
msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo))
>>>>>>> send18/dev
self._rc.memory.add(msg)
instruct_content[todo] = code
<<<<<<< HEAD
changed_files.add(coding_context.code_doc.filename)
if not changed_files:
logger.info("Nothing has changed.")
@ -140,8 +247,22 @@ class Engineer(Role):
cause_by=WriteCodeReview if self.use_code_review else WriteCode,
send_to=self,
sent_from=self,
=======
# code_msg = todo + FILENAME_CODE_SEP + str(file_path)
code_msg = (todo, file_path)
code_msg_all.append(code_msg)
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
instruct_content=instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
send_to="QaEngineer",
>>>>>>> send18/dev
)
<<<<<<< HEAD
async def _act_summarize(self):
code_summaries_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_FILE_REPO)
code_summaries_pdf_file_repo = CONFIG.git_repo.new_file_repository(CODE_SUMMARIES_PDF_FILE_REPO)
@ -232,6 +353,49 @@ class Engineer(Role):
async def _new_coding_doc(filename, src_file_repo, task_file_repo, design_file_repo, dependency):
context = await Engineer._new_coding_context(
filename, src_file_repo, task_file_repo, design_file_repo, dependency
=======
async def _act_sp_precision(self) -> Message:
code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later
instruct_content = {}
for todo in self.todos:
"""
# 从历史信息中挑选必须的信息，以减少prompt长度（人工经验总结）
1. Architect全部
2. ProjectManager全部
3. 是否需要其他代码（暂时需要）？
TODO:目标是不需要。在任务拆分清楚后，根据设计思路，不需要其他代码也能够写清楚单个文件，如果不能则表示还需要在定义的更清晰，这个是代码能够写长的关键
"""
context = []
msg = self._rc.memory.get_by_actions([WriteDesign, WriteTasks, WriteCode])
for m in msg:
context.append(m.content)
context_str = "\n".join(context)
# 编写code
code = await WriteCode().run(context=context_str, filename=todo)
# code review
if self.use_code_review:
try:
rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo)
code = rewrite_code
except Exception as e:
logger.error("code review failed!", e)
pass
file_path = await self.write_file(todo, code)
msg = Message(content=code, role=self.profile, cause_by=WriteCode)
self._rc.memory.add(msg)
instruct_content[todo] = code
code_msg = (todo, file_path)
code_msg_all.append(code_msg)
logger.info(f"Done {self.get_workspace()} generating.")
msg = Message(
content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all),
instruct_content=instruct_content,
role=self.profile,
cause_by=type(self._rc.todo),
send_to="QaEngineer",
>>>>>>> send18/dev
)
coding_doc = Document(root_path=str(src_file_repo.root_path), filename=filename, content=context.json())
return coding_doc

34
metagpt/roles/qa_engineer.py
View file

@ -14,6 +14,7 @@
@Modified By: mashenquan, 2023-12-5. Enhance the workflow to navigate to WriteCode or QaEngineer based on the results
of SummarizeCode.
"""
<<<<<<< HEAD
from metagpt.actions import DebugError, RunCode, WriteCode, WriteCodeReview, WriteTest
# from metagpt.const import WORKSPACE_ROOT
@ -24,6 +25,13 @@ from metagpt.const import (
TEST_CODES_FILE_REPO,
TEST_OUTPUTS_FILE_REPO,
)
=======
import os
from pathlib import Path
from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest
from metagpt.config import CONFIG
>>>>>>> send18/dev
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Document, Message, RunCodeContext, TestingContext
@ -47,6 +55,32 @@ class QaEngineer(Role):
self.test_round = 0
self.test_round_allowed = test_round_allowed
<<<<<<< HEAD
=======
@classmethod
def parse_workspace(cls, system_design_msg: Message) -> str:
if not system_design_msg.instruct_content:
return system_design_msg.instruct_content.dict().get("Python package name")
return CodeParser.parse_str(block="Python package name", text=system_design_msg.content)
def get_workspace(self, return_proj_dir=True) -> Path:
msg = self._rc.memory.get_by_action(WriteDesign)[-1]
if not msg:
return CONFIG.workspace / "src"
workspace = self.parse_workspace(msg)
# project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc.
if return_proj_dir:
return CONFIG.workspace / workspace
# development codes directory: workspace/{package_name}/{package_name}
return CONFIG.workspace / workspace / workspace
def write_file(self, filename: str, code: str):
workspace = self.get_workspace() / "tests"
file = workspace / filename
file.parent.mkdir(parents=True, exist_ok=True)
file.write_text(code)
>>>>>>> send18/dev
async def _write_test(self, message: Message) -> None:
src_file_repo = CONFIG.git_repo.new_file_repository(CONFIG.src_workspace)
changed_files = set(src_file_repo.changed_files.keys())

20
metagpt/roles/researcher.py
View file

@ -1,9 +1,15 @@
#!/usr/bin/env python
"""
<<<<<<< HEAD
@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116, change the data type of
the `cause_by` value in the `Message` to a string to support the new message distribution feature.
"""
=======
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
"""
>>>>>>> send18/dev
import asyncio
@ -41,6 +47,20 @@ class Researcher(Role):
if language not in ("en-us", "zh-cn"):
logger.warning(f"The language `{language}` has not been tested, it may not work.")
<<<<<<< HEAD
=======
async def _think(self) -> bool:
if self._rc.todo is None:
self._set_state(0)
return True
if self._rc.state + 1 < len(self._states):
self._set_state(self._rc.state + 1)
else:
self._rc.todo = None
return False
>>>>>>> send18/dev
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
todo = self._rc.todo

162
metagpt/roles/role.py
View file

@ -4,6 +4,7 @@
@Time : 2023/5/11 14:42
@Author : alexanderwu
@File : role.py
<<<<<<< HEAD
@Modified By: mashenquan, 2023-11-1. According to Chapter 2.2.1 and 2.2.2 of RFC 116:
1. Merge the `recv` functionality into the `_observe` function. Future message reading operations will be
consolidated within the `_observe` function.
@ -17,6 +18,10 @@
only. In the normal workflow, you should use `publish_message` or `put_message` to transmit messages.
@Modified By: mashenquan, 2023-11-4. According to the routing feature plan in Chapter 2.2.3.2 of RFC 113, the routing
functionality is to be consolidated into the `Environment` class.
=======
@Modified By: mashenquan, 2023-8-7, Support template-style variables, such as '{teaching_language} Teacher'.
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
>>>>>>> send18/dev
"""
from __future__ import annotations
@ -27,11 +32,19 @@ from pydantic import BaseModel, Field
from metagpt.actions import Action, ActionOutput
from metagpt.config import CONFIG
<<<<<<< HEAD
from metagpt.llm import LLM, HumanProvider
from metagpt.logs import logger
from metagpt.memory import Memory
from metagpt.schema import Message, MessageQueue
from metagpt.utils.common import any_to_name, any_to_str
=======
from metagpt.const import OPTIONS
from metagpt.llm import LLMFactory
from metagpt.logs import logger
from metagpt.memory import LongTermMemory, Memory
from metagpt.schema import Message, MessageTag
>>>>>>> send18/dev
PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """
@ -74,7 +87,11 @@ class RoleReactMode(str, Enum):
class RoleSetting(BaseModel):
<<<<<<< HEAD
"""Role Settings"""
=======
"""Role properties"""
>>>>>>> send18/dev
name: str
profile: str
@ -91,10 +108,16 @@ class RoleSetting(BaseModel):
class RoleContext(BaseModel):
<<<<<<< HEAD
"""Role Runtime Context"""
env: "Environment" = Field(default=None)
msg_buffer: MessageQueue = Field(default_factory=MessageQueue) # Message Buffer with Asynchronous Updates
=======
"""Runtime role context"""
env: "Environment" = Field(default=None)
>>>>>>> send18/dev
memory: Memory = Field(default_factory=Memory)
# long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory)
state: int = Field(default=-1) # -1 indicates initial or termination state where todo is None
@ -110,21 +133,34 @@ class RoleContext(BaseModel):
arbitrary_types_allowed = True
def check(self, role_id: str):
if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory:
if CONFIG.long_term_memory:
self.long_term_memory.recover_memory(role_id, self)
self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation
@property
def important_memory(self) -> list[Message]:
<<<<<<< HEAD
"""Get the information corresponding to the watched actions"""
=======
"""Retrieve information corresponding to the attention action."""
>>>>>>> send18/dev
return self.memory.get_by_actions(self.watch)
@property
def history(self) -> list[Message]:
return self.memory.get()
@property
def prerequisite(self):
"""Retrieve information with `prerequisite` tag"""
if self.memory and hasattr(self.memory, "get_by_tags"):
vv = self.memory.get_by_tags([MessageTag.Prerequisite.value])
return vv[-1:] if len(vv) > 1 else vv
return []
class Role:
<<<<<<< HEAD
"""Role/Agent"""
def __init__(self, name="", profile="", goal="", constraints="", desc="", is_human=False):
@ -132,6 +168,20 @@ class Role:
self._setting = RoleSetting(
name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, is_human=is_human
)
=======
"""Role/Proxy"""
def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs):
# Replace template-style variables, such as '{teaching_language} Teacher'.
name = Role.format_value(name)
profile = Role.format_value(profile)
goal = Role.format_value(goal)
constraints = Role.format_value(constraints)
desc = Role.format_value(desc)
self._llm = LLMFactory.new_llm()
self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc)
>>>>>>> send18/dev
self._states = []
self._actions = []
self._role_id = str(self._setting)
@ -208,8 +258,12 @@ class Role:
self._rc.todo = self._actions[self._rc.state] if state >= 0 else None
def set_env(self, env: "Environment"):
<<<<<<< HEAD
"""Set the environment in which the role works. The role can talk to the environment and can also receive
messages by observing."""
=======
"""设置角色工作所处的环境，角色可以向环境说话，也可以通过观察接受环境消息"""
>>>>>>> send18/dev
self._rc.env = env
if env:
env.set_subscription(self, self._subscription)
@ -221,6 +275,7 @@ class Role:
@property
def name(self):
<<<<<<< HEAD
"""Get virtual user name"""
return self._setting.name
@ -228,6 +283,30 @@ class Role:
def subscription(self) -> Set:
"""The labels for messages to be consumed by the Role object."""
return self._subscription
=======
"""Return role `name`, read only"""
return self._setting.name
@property
def desc(self):
"""Return role `desc`, read only"""
return self._setting.desc
@property
def goal(self):
"""Return role `goal`, read only"""
return self._setting.goal
@property
def constraints(self):
"""Return role `constraints`, read only"""
return self._setting.constraints
@property
def action_count(self):
"""Return number of action"""
return len(self._actions)
>>>>>>> send18/dev
def _get_prefix(self):
"""Get the role prefix"""
@ -235,14 +314,20 @@ class Role:
return self._setting.desc
return PREFIX_TEMPLATE.format(**self._setting.dict())
<<<<<<< HEAD
async def _think(self) -> None:
"""Think about what to do and decide on the next action"""
=======
async def _think(self) -> bool:
"""Consider what to do and decide on the next course of action. Return false if nothing can be done."""
>>>>>>> send18/dev
if len(self._actions) == 1:
# If there is only one action, then only this one can be performed
self._set_state(0)
return
return True
prompt = self._get_prefix()
prompt += STATE_TEMPLATE.format(
<<<<<<< HEAD
history=self._rc.history,
states="\n".join(self._states),
n_states=len(self._states) - 1,
@ -259,26 +344,49 @@ class Role:
if next_state == -1:
logger.info(f"End actions with {next_state=}")
self._set_state(next_state)
=======
history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1
)
next_state = await self._llm.aask(prompt)
logger.debug(f"{prompt=}")
if not next_state.isdigit() or int(next_state) not in range(len(self._states)):
logger.warning(f"Invalid answer of state, {next_state=}")
next_state = "0"
self._set_state(int(next_state))
return True
>>>>>>> send18/dev
async def _act(self) -> Message:
logger.info(f"{self._setting}: ready to {self._rc.todo}")
<<<<<<< HEAD
response = await self._rc.todo.run(self._rc.important_memory)
=======
requirement = self._rc.important_memory or self._rc.prerequisite
response = await self._rc.todo.run(requirement)
# logger.info(response)
>>>>>>> send18/dev
if isinstance(response, ActionOutput):
msg = Message(
content=response.content,
instruct_content=response.instruct_content,
role=self.profile,
<<<<<<< HEAD
cause_by=self._rc.todo,
sent_from=self,
)
elif isinstance(response, Message):
msg = response
=======
cause_by=type(self._rc.todo),
)
>>>>>>> send18/dev
else:
msg = Message(content=response, role=self.profile, cause_by=self._rc.todo, sent_from=self)
self._rc.memory.add(msg)
return msg
<<<<<<< HEAD
async def _observe(self, ignore_memory=False) -> int:
"""Prepare new messages for processing from the message buffer and other sources."""
# Read unprocessed messages from the msg buffer.
@ -292,6 +400,21 @@ class Role:
# Design Rules:
# If you need to further categorize Message objects, you can do so using the Message.set_meta function.
# msg_buffer is a receiving buffer, avoid adding message data and operations to msg_buffer.
=======
async def _observe(self) -> int:
"""从环境中观察，获得重要信息，并加入记忆"""
if not self._rc.env:
return 0
env_msgs = self._rc.env.memory.get()
observed = self._rc.env.memory.get_by_actions(self._rc.watch)
self._rc.news = self._rc.memory.remember(observed) # remember recent exact or similar memories
for i in env_msgs:
self.recv(i)
>>>>>>> send18/dev
news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news]
if news_text:
logger.debug(f"{self._setting} observed: {news_text}")
@ -382,10 +505,36 @@ class Role:
self.publish_message(rsp)
return rsp
<<<<<<< HEAD
@property
def is_idle(self) -> bool:
"""If true, all actions have been executed."""
return not self._rc.news and not self._rc.todo and self._rc.msg_buffer.empty()
=======
@staticmethod
def format_value(value):
"""Fill parameters inside `value` with `options`."""
if not isinstance(value, str):
return value
if "{" not in value:
return value
merged_opts = OPTIONS.get() or {}
try:
return value.format(**merged_opts)
except KeyError as e:
logger.warning(f"Parameter is missing:{e}")
for k, v in merged_opts.items():
value = value.replace("{" + f"{k}" + "}", str(v))
return value
def add_action(self, act):
self._actions.append(act)
def add_to_do(self, act):
self._rc.todo = act
>>>>>>> send18/dev
async def think(self) -> Action:
"""The exported `think` function"""
@ -398,7 +547,16 @@ class Role:
return ActionOutput(content=msg.content, instruct_content=msg.instruct_content)
@property
<<<<<<< HEAD
def todo(self) -> str:
if self._actions:
return any_to_name(self._actions[0])
return ""
=======
def todo_description(self):
if not self._rc or not self._rc.todo:
return ""
if self._rc.todo.desc:
return self._rc.todo.desc
return f"{type(self._rc.todo).__name__}"
>>>>>>> send18/dev

113
metagpt/roles/teacher.py Normal file
View file

@ -0,0 +1,113 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/7/27
@Author : mashenquan
@File : teacher.py
@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue.
"""
import re
import aiofiles
from metagpt.actions.write_teaching_plan import (
TeachingPlanRequirement,
WriteTeachingPlanPart,
)
from metagpt.config import CONFIG
from metagpt.logs import logger
from metagpt.roles import Role
from metagpt.schema import Message
class Teacher(Role):
"""Support configurable teacher roles,
with native and teaching languages being replaceable through configurations."""
def __init__(
self,
name="Lily",
profile="{teaching_language} Teacher",
goal="writing a {language} teaching plan part by part",
constraints="writing in {language}",
desc="",
*args,
**kwargs,
):
super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs)
actions = []
for topic in WriteTeachingPlanPart.TOPICS:
act = WriteTeachingPlanPart(topic=topic, llm=self._llm)
actions.append(act)
self._init_actions(actions)
self._watch({TeachingPlanRequirement})
async def _think(self) -> bool:
"""Everything will be done part by part."""
if self._rc.todo is None:
self._set_state(0)
return True
if self._rc.state + 1 < len(self._states):
self._set_state(self._rc.state + 1)
return True
self._rc.todo = None
return False
async def _react(self) -> Message:
ret = Message(content="")
while True:
await self._think()
if self._rc.todo is None:
break
logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}")
msg = await self._act()
if ret.content != "":
ret.content += "\n\n\n"
ret.content += msg.content
logger.info(ret.content)
await self.save(ret.content)
return ret
async def save(self, content):
"""Save teaching plan"""
filename = Teacher.new_file_name(self.course_title)
pathname = CONFIG.workspace / "teaching_plan"
pathname.mkdir(exist_ok=True)
pathname = pathname / filename
try:
async with aiofiles.open(str(pathname), mode="w", encoding="utf-8") as writer:
await writer.write(content)
except Exception as e:
logger.error(f"Save failed：{e}")
logger.info(f"Save to:{pathname}")
@staticmethod
def new_file_name(lesson_title, ext=".md"):
"""Create a related file name based on `lesson_title` and `ext`."""
# Define the special characters that need to be replaced.
illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']'
# Replace the special characters with underscores.
filename = re.sub(illegal_chars, "_", lesson_title) + ext
return re.sub(r"_+", "_", filename)
@property
def course_title(self):
"""Return course title of teaching plan"""
default_title = "teaching_plan"
for act in self._actions:
if act.topic != WriteTeachingPlanPart.COURSE_TITLE:
continue
if act.rsp is None:
return default_title
title = act.rsp.lstrip("# \n")
if "\n" in title:
ix = title.index("\n")
title = title[0:ix]
return title
return default_title

3
metagpt/schema.py
View file

@ -22,9 +22,7 @@ from asyncio import Queue, QueueEmpty, wait_for
from json import JSONDecodeError
from pathlib import Path
from typing import Dict, List, Optional, Set, TypedDict
from pydantic import BaseModel, Field
from metagpt.config import CONFIG
from metagpt.const import (
MESSAGE_ROUTE_CAUSE_BY,
@ -345,3 +343,4 @@ class CodeSummarizeContext(BaseModel):
class BugFixContext(BaseModel):
filename: str = ""

8
metagpt/team.py
View file

@ -42,9 +42,11 @@ class Team(BaseModel):
CONFIG.max_budget = investment
logger.info(f"Investment: ${investment}.")
def _check_balance(self):
if CONFIG.total_cost > CONFIG.max_budget:
raise NoMoneyException(CONFIG.total_cost, f"Insufficient funds: {CONFIG.max_budget}")
@staticmethod
def _check_balance():
if CONFIG.cost_manager.total_cost > CONFIG.cost_manager.max_budget:
raise NoMoneyException(CONFIG.cost_manager.total_cost,
f'Insufficient funds: {CONFIG.cost_manager.max_budget}')
def run_project(self, idea, send_to: str = ""):
"""Start a project from publishing user requirement."""

5
metagpt/tools/__init__.py
View file

@ -22,3 +22,8 @@ class WebBrowserEngineType(Enum):
PLAYWRIGHT = "playwright"
SELENIUM = "selenium"
CUSTOM = "custom"
@classmethod
def _missing_(cls, key):
"""缺省类型转换"""
return cls.CUSTOM

113
metagpt/tools/azure_tts.py Normal file
View file

@ -0,0 +1,113 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/17
@Author : mashenquan
@File : azure_tts.py
@Desc : azure TTS OAS3 api, which provides text-to-speech functionality
"""
import asyncio
import base64
from pathlib import Path
from uuid import uuid4
import aiofiles
from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer
from metagpt.config import CONFIG, Config
from metagpt.logs import logger
class AzureTTS:
"""Azure Text-to-Speech"""
def __init__(self, subscription_key, region):
"""
:param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint`
:param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API.
"""
self.subscription_key = subscription_key if subscription_key else CONFIG.AZURE_TTS_SUBSCRIPTION_KEY
self.region = region if region else CONFIG.AZURE_TTS_REGION
# 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles
async def synthesize_speech(self, lang, voice, text, output_file):
speech_config = SpeechConfig(subscription=self.subscription_key, region=self.region)
speech_config.speech_synthesis_voice_name = voice
audio_config = AudioConfig(filename=output_file)
synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config)
# More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice
ssml_string = (
"<speak version='1.0' xmlns='http://www.w3.org/2001/10/synthesis' "
f"xml:lang='{lang}' xmlns:mstts='http://www.w3.org/2001/mstts'>"
f"<voice name='{voice}'>{text}</voice></speak>"
)
return synthesizer.speak_ssml_async(ssml_string).get()
@staticmethod
def role_style_text(role, style, text):
return f'<mstts:express-as role="{role}" style="{style}">{text}</mstts:express-as>'
@staticmethod
def role_text(role, text):
return f'<mstts:express-as role="{role}">{text}</mstts:express-as>'
@staticmethod
def style_text(style, text):
return f'<mstts:express-as style="{style}">{text}</mstts:express-as>'
# Export
async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""):
"""Text to speech
For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery`
:param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`
:param text: The text used for voice conversion.
:param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint`
:param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API.
:return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string.
"""
if not text:
return ""
if not lang:
lang = "zh-CN"
if not voice:
voice = "zh-CN-XiaomoNeural"
if not role:
role = "Girl"
if not style:
style = "affectionate"
if not subscription_key:
subscription_key = CONFIG.AZURE_TTS_SUBSCRIPTION_KEY
if not region:
region = CONFIG.AZURE_TTS_REGION
xml_value = AzureTTS.role_style_text(role=role, style=style, text=text)
tts = AzureTTS(subscription_key=subscription_key, region=region)
filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav")
try:
await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename))
async with aiofiles.open(filename, mode="rb") as reader:
data = await reader.read()
base64_string = base64.b64encode(data).decode("utf-8")
filename.unlink()
except Exception as e:
logger.error(f"text:{text}, error:{e}")
return ""
return base64_string
if __name__ == "__main__":
Config()
loop = asyncio.new_event_loop()
v = loop.create_task(oas3_azsure_tts("测试，test"))
loop.run_until_complete(v)
print(v)

27
metagpt/tools/hello.py Normal file
View file

@ -0,0 +1,27 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/5/2 16:03
@Author : mashenquan
@File : hello.py
@Desc : Implement the OpenAPI Specification 3.0 demo and use the following command to test the HTTP service:
curl -X 'POST' \
'http://localhost:8080/openapi/greeting/dave' \
-H 'accept: text/plain' \
-H 'Content-Type: application/json' \
-d '{}'
"""
import connexion
# openapi implement
async def post_greeting(name: str) -> str:
return f"Hello {name}\n"
if __name__ == "__main__":
app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/')
app.add_api("openapi.yaml", arguments={"title": "Hello World Example"})
app.run(port=8080)

162
metagpt/tools/iflytek_tts.py Normal file
View file

@ -0,0 +1,162 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/17
@Author : mashenquan
@File : iflytek_tts.py
@Desc : iFLYTEK TTS OAS3 api, which provides text-to-speech functionality
"""
import asyncio
import base64
import hashlib
import hmac
import json
import uuid
from datetime import datetime
from enum import Enum
from pathlib import Path
from time import mktime
from typing import Optional
from urllib.parse import urlencode
from wsgiref.handlers import format_date_time
import aiofiles
import websockets as websockets
from pydantic import BaseModel
from metagpt.config import CONFIG
from metagpt.logs import logger
class IFlyTekTTSStatus(Enum):
STATUS_FIRST_FRAME = 0 # The first frame
STATUS_CONTINUE_FRAME = 1 # The intermediate frame
STATUS_LAST_FRAME = 2 # The last frame
class AudioData(BaseModel):
audio: str
status: int
ced: str
class IFlyTekTTSResponse(BaseModel):
code: int
message: str
data: Optional[AudioData] = None
sid: str
DEFAULT_IFLYTEK_VOICE = "xiaoyan"
class IFlyTekTTS(object):
def __init__(self, app_id: str, api_key: str, api_secret: str):
"""
:param app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`
:param api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
:param api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
"""
self.app_id = app_id or CONFIG.IFLYTEK_APP_ID
self.api_key = api_key or CONFIG.IFLYTEK_API_KEY
self.api_secret = api_secret or CONFIG.API_SECRET
async def synthesize_speech(self, text, output_file: str, voice=DEFAULT_IFLYTEK_VOICE):
url = self._create_url()
data = {
"common": {"app_id": self.app_id},
"business": {"aue": "lame", "sfl": 1, "auf": "audio/L16;rate=16000", "vcn": voice, "tte": "utf8"},
"data": {"status": 2, "text": str(base64.b64encode(text.encode("utf-8")), "UTF8")},
}
req = json.dumps(data)
async with websockets.connect(url) as websocket:
# send request
await websocket.send(req)
# receive frames
async with aiofiles.open(str(output_file), "w") as writer:
while True:
v = await websocket.recv()
rsp = IFlyTekTTSResponse(**json.loads(v))
if rsp.data:
await writer.write(rsp.data.audio)
if rsp.data.status != IFlyTekTTSStatus.STATUS_LAST_FRAME.value:
continue
break
def _create_url(self):
"""Create request url"""
url = "wss://tts-api.xfyun.cn/v2/tts"
# Generate a timestamp in RFC1123 format
now = datetime.now()
date = format_date_time(mktime(now.timetuple()))
signature_origin = "host: " + "ws-api.xfyun.cn" + "\n"
signature_origin += "date: " + date + "\n"
signature_origin += "GET " + "/v2/tts " + "HTTP/1.1"
# Perform HMAC-SHA256 encryption
signature_sha = hmac.new(
self.api_secret.encode("utf-8"), signature_origin.encode("utf-8"), digestmod=hashlib.sha256
).digest()
signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8")
authorization_origin = 'api_key="%s", algorithm="%s", headers="%s", signature="%s"' % (
self.api_key,
"hmac-sha256",
"host date request-line",
signature_sha,
)
authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8")
# Combine the authentication parameters of the request into a dictionary.
v = {"authorization": authorization, "date": date, "host": "ws-api.xfyun.cn"}
# Concatenate the authentication parameters to generate the URL.
url = url + "?" + urlencode(v)
return url
# Export
async def oas3_iflytek_tts(text: str, voice: str = "", app_id: str = "", api_key: str = "", api_secret: str = ""):
"""Text to speech
For more details, check out:`https://www.xfyun.cn/doc/tts/online_tts/API.html`
:param voice: Default `xiaoyan`. For more details, checkout: `https://www.xfyun.cn/doc/tts/online_tts/API.html#%E6%8E%A5%E5%8F%A3%E8%B0%83%E7%94%A8%E6%B5%81%E7%A8%8B`
:param text: The text used for voice conversion.
:param app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`
:param api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
:param api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts`
:return: Returns the Base64-encoded .mp3 file data if successful, otherwise an empty string.
"""
if not app_id:
app_id = CONFIG.IFLYTEK_APP_ID
if not api_key:
api_key = CONFIG.IFLYTEK_API_KEY
if not api_secret:
api_secret = CONFIG.IFLYTEK_API_SECRET
if not voice:
voice = CONFIG.IFLYTEK_VOICE or DEFAULT_IFLYTEK_VOICE
filename = Path(__file__).parent / (uuid.uuid4().hex + ".mp3")
try:
tts = IFlyTekTTS(app_id=app_id, api_key=api_key, api_secret=api_secret)
await tts.synthesize_speech(text=text, output_file=str(filename), voice=voice)
async with aiofiles.open(str(filename), mode="r") as reader:
base64_string = await reader.read()
except Exception as e:
logger.error(f"text:{text}, error:{e}")
base64_string = ""
finally:
filename.unlink()
return base64_string
if __name__ == "__main__":
asyncio.get_event_loop().run_until_complete(
oas3_iflytek_tts(
text="你好，hello",
app_id="f7acef62",
api_key="fda72e3aa286042a492525816a5efa08",
api_secret="ZDk3NjdiMDBkODJlOWQ1NjRjMGI2NDY4",
)
)

44
metagpt/tools/metagpt_oas3_api_svc.py Normal file
View file

@ -0,0 +1,44 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/17
@Author : mashenquan
@File : metagpt_oas3_api_svc.py
@Desc : MetaGPT OpenAPI Specification 3.0 REST API service
"""
import asyncio
import sys
from pathlib import Path
import connexion
sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt'
def oas_http_svc():
"""Start the OAS 3.0 OpenAPI HTTP service"""
app = connexion.AioHttpApp(__name__, specification_dir="../../.well-known/")
app.add_api("metagpt_oas3_api.yaml")
app.add_api("openapi.yaml")
app.run(port=8080)
async def async_main():
"""Start the OAS 3.0 OpenAPI HTTP service in the background."""
loop = asyncio.get_event_loop()
loop.run_in_executor(None, oas_http_svc)
# TODO: replace following codes:
while True:
await asyncio.sleep(1)
print("sleep")
def main():
print("http://localhost:8080/oas3/ui/")
oas_http_svc()
if __name__ == "__main__":
# asyncio.run(async_main())
main()

117
metagpt/tools/metagpt_text_to_image.py Normal file
View file

@ -0,0 +1,117 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/18
@Author : mashenquan
@File : metagpt_text_to_image.py
@Desc : MetaGPT Text-to-Image OAS3 api, which provides text-to-image functionality.
"""
import asyncio
import base64
import os
import sys
from pathlib import Path
from typing import List, Dict
import aiohttp
import requests
from pydantic import BaseModel
from metagpt.config import CONFIG, Config
sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt'
from metagpt.logs import logger
class MetaGPTText2Image:
def __init__(self, model_url):
"""
:param model_url: Model reset api url
"""
self.model_url = model_url if model_url else CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL
async def text_2_image(self, text, size_type="512x512"):
"""Text to image
:param text: The text used for image conversion.
:param size_type: One of ['512x512', '512x768']
:return: The image data is returned in Base64 encoding.
"""
headers = {
"Content-Type": "application/json"
}
dims = size_type.split("x")
data = {
"prompt": text,
"negative_prompt": "(easynegative:0.8),black, dark,Low resolution",
"override_settings": {"sd_model_checkpoint": "galaxytimemachinesGTM_photoV20"},
"seed": -1,
"batch_size": 1,
"n_iter": 1,
"steps": 20,
"cfg_scale": 11,
"width": int(dims[0]),
"height": int(dims[1]), # 768,
"restore_faces": False,
"tiling": False,
"do_not_save_samples": False,
"do_not_save_grid": False,
"enable_hr": False,
"hr_scale": 2,
"hr_upscaler": "Latent",
"hr_second_pass_steps": 0,
"hr_resize_x": 0,
"hr_resize_y": 0,
"hr_upscale_to_x": 0,
"hr_upscale_to_y": 0,
"truncate_x": 0,
"truncate_y": 0,
"applied_old_hires_behavior_to": None,
"eta": None,
"sampler_index": "DPM++ SDE Karras",
"alwayson_scripts": {},
}
class ImageResult(BaseModel):
images: List
parameters: Dict
try:
async with aiohttp.ClientSession() as session:
async with session.post(self.model_url, headers=headers, json=data) as response:
result = ImageResult(**await response.json())
if len(result.images) == 0:
return ""
return result.images[0]
except requests.exceptions.RequestException as e:
logger.error(f"An error occurred:{e}")
return ""
# Export
async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""):
"""Text to image
:param text: The text used for image conversion.
:param model_url: Model reset api
:param size_type: One of ['512x512', '512x768']
:return: The image data is returned in Base64 encoding.
"""
if not text:
return ""
if not model_url:
model_url = CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL
return await MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type)
if __name__ == "__main__":
Config()
loop = asyncio.new_event_loop()
task = loop.create_task(oas3_metagpt_text_to_image("Panda emoji"))
v = loop.run_until_complete(task)
print(v)
data = base64.b64decode(v)
with open("tmp.png", mode="wb") as writer:
writer.write(data)
print(v)

96
metagpt/tools/openai_text_to_embedding.py Normal file
View file

@ -0,0 +1,96 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/18
@Author : mashenquan
@File : openai_text_to_embedding.py
@Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality.
For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object`
"""
import asyncio
import os
from pathlib import Path
from typing import List
import aiohttp
import requests
from pydantic import BaseModel
import sys
from metagpt.config import CONFIG, Config
sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt'
from metagpt.logs import logger
class Embedding(BaseModel):
"""Represents an embedding vector returned by embedding endpoint."""
object: str # The object type, which is always "embedding".
embedding: List[
float] # The embedding vector, which is a list of floats. The length of vector depends on the model as listed in the embedding guide.
index: int # The index of the embedding in the list of embeddings.
class Usage(BaseModel):
prompt_tokens: int
total_tokens: int
class ResultEmbedding(BaseModel):
object: str
data: List[Embedding]
model: str
usage: Usage
class OpenAIText2Embedding:
def __init__(self, openai_api_key):
"""
:param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
"""
self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY
async def text_2_embedding(self, text, model="text-embedding-ada-002"):
"""Text to embedding
:param text: The text used for embedding.
:param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`.
:return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`.
"""
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {self.openai_api_key}"
}
data = {"input": text, "model": model}
try:
async with aiohttp.ClientSession() as session:
async with session.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) as response:
return await response.json()
except requests.exceptions.RequestException as e:
logger.error(f"An error occurred:{e}")
return {}
# Export
async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""):
"""Text to embedding
:param text: The text used for embedding.
:param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`.
:param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
:return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`.
"""
if not text:
return ""
if not openai_api_key:
openai_api_key = CONFIG.OPENAI_API_KEY
return await OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model)
if __name__ == "__main__":
Config()
loop = asyncio.new_event_loop()
task = loop.create_task(oas3_openai_text_to_embedding("Panda emoji"))
v = loop.run_until_complete(task)
print(v)

93
metagpt/tools/openai_text_to_image.py Normal file
View file

@ -0,0 +1,93 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/17
@Author : mashenquan
@File : openai_text_to_image.py
@Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality.
"""
import asyncio
import base64
import aiohttp
import openai
import requests
from metagpt.config import CONFIG, Config
from metagpt.logs import logger
class OpenAIText2Image:
def __init__(self, openai_api_key):
"""
:param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
"""
self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY
async def text_2_image(self, text, size_type="1024x1024"):
"""Text to image
:param text: The text used for image conversion.
:param size_type: One of ['256x256', '512x512', '1024x1024']
:return: The image data is returned in Base64 encoding.
"""
try:
result = await openai.Image.acreate(
api_key=CONFIG.OPENAI_API_KEY,
api_base=CONFIG.OPENAI_API_BASE,
api_type=None,
api_version=None,
organization=None,
prompt=text,
n=1,
size=size_type,
)
except Exception as e:
logger.error(f"An error occurred:{e}")
return ""
if result and len(result.data) > 0:
return await OpenAIText2Image.get_image_data(result.data[0].url)
return ""
@staticmethod
async def get_image_data(url):
"""Fetch image data from a URL and encode it as Base64
:param url: Image url
:return: Base64-encoded image data.
"""
try:
async with aiohttp.ClientSession() as session:
async with session.get(url) as response:
response.raise_for_status() # 如果是 4xx 或 5xx 响应，会引发异常
image_data = await response.read()
base64_image = base64.b64encode(image_data).decode("utf-8")
return base64_image
except requests.exceptions.RequestException as e:
logger.error(f"An error occurred:{e}")
return ""
# Export
async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""):
"""Text to image
:param text: The text used for image conversion.
:param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`
:param size_type: One of ['256x256', '512x512', '1024x1024']
:return: The image data is returned in Base64 encoding.
"""
if not text:
return ""
if not openai_api_key:
openai_api_key = CONFIG.OPENAI_API_KEY
return await OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type)
if __name__ == "__main__":
Config()
loop = asyncio.new_event_loop()
task = loop.create_task(oas3_openai_text_to_image("Panda emoji"))
v = loop.run_until_complete(task)
print(v)

9
metagpt/tools/sd_engine.py
View file

@ -13,7 +13,12 @@ from typing import List
from aiohttp import ClientSession
from PIL import Image, PngImagePlugin
<<<<<<< HEAD
from metagpt.config import CONFIG
=======
from metagpt.config import Config
from metagpt.logs import logger
>>>>>>> send18/dev
# from metagpt.const import WORKSPACE_ROOT
from metagpt.logs import logger
@ -79,7 +84,11 @@ class SDEngine:
return self.payload
def _save(self, imgs, save_name=""):
<<<<<<< HEAD
save_dir = CONFIG.workspace_path / "resources" / "SD_Output"
=======
save_dir = CONFIG.get_workspace() / "resources" / "SD_Output"
>>>>>>> send18/dev
if not os.path.exists(save_dir):
os.makedirs(save_dir, exist_ok=True)
batch_decode_base64_to_image(imgs, save_dir, save_name=save_name)

24
metagpt/tools/web_browser_engine.py
View file

@ -1,9 +1,12 @@
#!/usr/bin/env python
"""
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
from __future__ import annotations
import importlib
from typing import Any, Callable, Coroutine, Literal, overload
from typing import Any, Callable, Coroutine, Dict, Literal, overload
from metagpt.config import CONFIG
from metagpt.tools import WebBrowserEngineType
@ -13,18 +16,21 @@ from metagpt.utils.parse_html import WebPage
class WebBrowserEngine:
def __init__(
self,
options: Dict,
engine: WebBrowserEngineType | None = None,
run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None,
):
engine = engine or CONFIG.web_browser_engine
engine = engine or options.get("web_browser_engine")
if engine is None:
raise NotImplementedError
if engine == WebBrowserEngineType.PLAYWRIGHT:
if WebBrowserEngineType(engine) is WebBrowserEngineType.PLAYWRIGHT:
module = "metagpt.tools.web_browser_engine_playwright"
run_func = importlib.import_module(module).PlaywrightWrapper().run
elif engine == WebBrowserEngineType.SELENIUM:
run_func = importlib.import_module(module).PlaywrightWrapper(options=options).run
elif WebBrowserEngineType(engine) is WebBrowserEngineType.SELENIUM:
module = "metagpt.tools.web_browser_engine_selenium"
run_func = importlib.import_module(module).SeleniumWrapper().run
elif engine == WebBrowserEngineType.CUSTOM:
run_func = importlib.import_module(module).SeleniumWrapper(options=options).run
elif WebBrowserEngineType(engine) is WebBrowserEngineType.CUSTOM:
run_func = run_func
else:
raise NotImplementedError
@ -47,6 +53,8 @@ if __name__ == "__main__":
import fire
async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs):
return await WebBrowserEngine(WebBrowserEngineType(engine_type), **kwargs).run(url, *urls)
return await WebBrowserEngine(options=CONFIG.options, engine=WebBrowserEngineType(engine_type), **kwargs).run(
url, *urls
)
fire.Fire(main)

6
metagpt/tools/web_browser_engine_playwright.py
View file

@ -1,4 +1,8 @@
#!/usr/bin/env python
"""
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
from __future__ import annotations
import asyncio
@ -144,6 +148,6 @@ if __name__ == "__main__":
import fire
async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs):
return await PlaywrightWrapper(browser_type, **kwargs).run(url, *urls)
return await PlaywrightWrapper(browser_type=browser_type, **kwargs).run(url, *urls)
fire.Fire(main)

19
metagpt/tools/web_browser_engine_selenium.py
View file

@ -1,17 +1,21 @@
#!/usr/bin/env python
"""
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
from __future__ import annotations
import asyncio
import importlib
from concurrent import futures
from copy import deepcopy
from typing import Literal
from typing import Literal, Dict
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait
from metagpt.config import CONFIG
from metagpt.config import Config
from metagpt.utils.parse_html import WebPage
@ -29,6 +33,7 @@ class SeleniumWrapper:
def __init__(
self,
options: Dict,
browser_type: Literal["chrome", "firefox", "edge", "ie"] | None = None,
launch_kwargs: dict | None = None,
*,
@ -36,11 +41,11 @@ class SeleniumWrapper:
executor: futures.Executor | None = None,
) -> None:
if browser_type is None:
browser_type = CONFIG.selenium_browser_type
browser_type = options.get("selenium_browser_type")
self.browser_type = browser_type
launch_kwargs = launch_kwargs or {}
if CONFIG.global_proxy and "proxy-server" not in launch_kwargs:
launch_kwargs["proxy-server"] = CONFIG.global_proxy
if options.get("global_proxy") and "proxy-server" not in launch_kwargs:
launch_kwargs["proxy-server"] = options.get("global_proxy")
self.executable_path = launch_kwargs.pop("executable_path", None)
self.launch_args = [f"--{k}={v}" for k, v in launch_kwargs.items()]
@ -118,6 +123,8 @@ if __name__ == "__main__":
import fire
async def main(url: str, *urls: str, browser_type: str = "chrome", **kwargs):
return await SeleniumWrapper(browser_type, **kwargs).run(url, *urls)
return await SeleniumWrapper(options=Config().runtime_options,
browser_type=browser_type,
**kwargs).run(url, *urls)
fire.Fire(main)

6
metagpt/utils/common.py
View file

@ -18,8 +18,10 @@ import os
import platform
import re
from typing import List, Tuple, Union
from metagpt.const import MESSAGE_ROUTE_TO_ALL
from pathlib import Path
from typing import List, Tuple
import yaml
from metagpt.logs import logger
@ -184,7 +186,7 @@ class OutputParser:
if start_index != -1 and end_index != -1:
# Extract the structure part
structure_text = text[start_index : end_index + 1]
structure_text = text[start_index: end_index + 1]
try:
# Attempt to convert the text to a Python data type using ast.literal_eval

79
metagpt/utils/cost_manager.py Normal file
View file

@ -0,0 +1,79 @@
# -*- coding: utf-8 -*-
"""
@Time : 2023/8/28
@Author : mashenquan
@File : openai.py
@Desc : mashenquan, 2023/8/28. Separate the `CostManager` class to support user-level cost accounting.
"""
from pydantic import BaseModel
from metagpt.logs import logger
from metagpt.utils.token_counter import TOKEN_COSTS
from typing import NamedTuple
class Costs(NamedTuple):
total_prompt_tokens: int
total_completion_tokens: int
total_cost: float
total_budget: float
class CostManager(BaseModel):
"""Calculate the overhead of using the interface."""
total_prompt_tokens: int = 0
total_completion_tokens: int = 0
total_budget: float = 0
max_budget: float = 10.0
total_cost: float = 0
def update_cost(self, prompt_tokens, completion_tokens, model):
"""
Update the total cost, prompt tokens, and completion tokens.
Args:
prompt_tokens (int): The number of tokens used in the prompt.
completion_tokens (int): The number of tokens used in the completion.
model (str): The model used for the API call.
"""
self.total_prompt_tokens += prompt_tokens
self.total_completion_tokens += completion_tokens
cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][
"completion"]) / 1000
self.total_cost += cost
logger.info(
f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | "
f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}"
)
def get_total_prompt_tokens(self):
"""
Get the total number of prompt tokens.
Returns:
int: The total number of prompt tokens.
"""
return self.total_prompt_tokens
def get_total_completion_tokens(self):
"""
Get the total number of completion tokens.
Returns:
int: The total number of completion tokens.
"""
return self.total_completion_tokens
def get_total_cost(self):
"""
Get the total cost of API calls.
Returns:
float: The total cost of API calls.
"""
return self.total_cost
def get_costs(self) -> Costs:
"""Get all costs"""
return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget)

146
metagpt/utils/mermaid.py
View file

@ -4,15 +4,25 @@
@Time : 2023/7/4 10:53
@Author : alexanderwu alitrack
@File : mermaid.py
@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation.
"""
import asyncio
<<<<<<< HEAD
import os
from pathlib import Path
from metagpt.config import CONFIG
from metagpt.const import METAGPT_ROOT
=======
from pathlib import Path
# from metagpt.utils.common import check_cmd_exists
import aiofiles
from metagpt.config import CONFIG, Config
from metagpt.const import PROJECT_ROOT
>>>>>>> send18/dev
from metagpt.logs import logger
from metagpt.utils.common import check_cmd_exists
async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int:
@ -29,8 +39,11 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
if dir_name and not os.path.exists(dir_name):
os.makedirs(dir_name)
tmp = Path(f"{output_file_without_suffix}.mmd")
tmp.write_text(mermaid_code, encoding="utf-8")
async with aiofiles.open(tmp, "w", encoding="utf-8") as f:
await f.write(mermaid_code)
# tmp.write_text(mermaid_code, encoding="utf-8")
<<<<<<< HEAD
engine = CONFIG.mermaid_engine.lower()
if engine == "nodejs":
if check_cmd_exists(CONFIG.mmdc) != 0:
@ -87,60 +100,93 @@ async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048,
logger.warning(f"Unsupported mermaid engine: {engine}")
return 0
=======
# if check_cmd_exists("mmdc") != 0:
# logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc")
# return -1
MMC1 = """classDiagram
class Main {
-SearchEngine search_engine
+main() str
}
class SearchEngine {
-Index index
-Ranking ranking
-Summary summary
+search(query: str) str
}
class Index {
-KnowledgeBase knowledge_base
+create_index(data: dict)
+query_index(query: str) list
}
class Ranking {
+rank_results(results: list) list
}
class Summary {
+summarize_results(results: list) str
}
class KnowledgeBase {
+update(data: dict)
+fetch_data(query: str) dict
}
Main --> SearchEngine
SearchEngine --> Index
SearchEngine --> Ranking
SearchEngine --> Summary
Index --> KnowledgeBase"""
# for suffix in ["pdf", "svg", "png"]:
for suffix in ["png"]:
output_file = f"{output_file_without_suffix}.{suffix}"
# Call the `mmdc` command to convert the Mermaid code to a PNG
logger.info(f"Generating {output_file}..")
cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]
MMC2 = """sequenceDiagram
participant M as Main
participant SE as SearchEngine
participant I as Index
participant R as Ranking
participant S as Summary
participant KB as KnowledgeBase
M->>SE: search(query)
SE->>I: query_index(query)
I->>KB: fetch_data(query)
KB-->>I: return data
I-->>SE: return results
SE->>R: rank_results(results)
R-->>SE: return ranked_results
SE->>S: summarize_results(ranked_results)
S-->>SE: return summary
SE-->>M: return summary"""
if CONFIG.puppeteer_config:
cmds.extend(["-p", CONFIG.puppeteer_config])
process = await asyncio.create_subprocess_exec(*cmds)
await process.wait()
return process.returncode
>>>>>>> send18/dev
if __name__ == "__main__":
MMC1 = """classDiagram
class Main {
-SearchEngine search_engine
+main() str
}
class SearchEngine {
-Index index
-Ranking ranking
-Summary summary
+search(query: str) str
}
class Index {
-KnowledgeBase knowledge_base
+create_index(data: dict)
+query_index(query: str) list
}
class Ranking {
+rank_results(results: list) list
}
class Summary {
+summarize_results(results: list) str
}
class KnowledgeBase {
+update(data: dict)
+fetch_data(query: str) dict
}
Main --> SearchEngine
SearchEngine --> Index
SearchEngine --> Ranking
SearchEngine --> Summary
Index --> KnowledgeBase"""
MMC2 = """sequenceDiagram
participant M as Main
participant SE as SearchEngine
participant I as Index
participant R as Ranking
participant S as Summary
participant KB as KnowledgeBase
M->>SE: search(query)
SE->>I: query_index(query)
I->>KB: fetch_data(query)
KB-->>I: return data
I-->>SE: return results
SE->>R: rank_results(results)
R-->>SE: return ranked_results
SE->>S: summarize_results(ranked_results)
S-->>SE: return summary
SE-->>M: return summary"""
<<<<<<< HEAD
if __name__ == "__main__":
loop = asyncio.new_event_loop()
result = loop.run_until_complete(mermaid_to_file(MMC1, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1"))
result = loop.run_until_complete(mermaid_to_file(MMC2, METAGPT_ROOT / f"{CONFIG.mermaid_engine}/1"))
loop.close()
=======
conf = Config()
asyncio.run(
mermaid_to_file(
options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png"
)
)
asyncio.run(
mermaid_to_file(
options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png"
)
)
>>>>>>> send18/dev

219
metagpt/utils/redis.py Normal file
View file

@ -0,0 +1,219 @@
# !/usr/bin/python3
# -*- coding: utf-8 -*-
# @Author: Hui
# @Desc: { redis client }
# @Date: 2022/11/28 10:12
import json
import traceback
from datetime import timedelta
from enum import Enum
from typing import Awaitable, Callable, Dict, Optional, Union
from redis import asyncio as aioredis
from metagpt.config import CONFIG
from metagpt.logs import logger
class RedisTypeEnum(Enum):
"""Redis 数据类型"""
String = "String"
List = "List"
Hash = "Hash"
Set = "Set"
ZSet = "ZSet"
def make_url(
dialect: str,
*,
user: Optional[str] = None,
password: Optional[str] = None,
host: Optional[str] = None,
port: Optional[Union[str, int]] = None,
name: Optional[Union[str, int]] = None,
) -> str:
url_parts = [f"{dialect}://"]
if user or password:
if user:
url_parts.append(user)
if password:
url_parts.append(f":{password}")
url_parts.append("@")
if not host and not dialect.startswith("sqlite"):
host = "127.0.0.1"
if host:
url_parts.append(f"{host}")
if port:
url_parts.append(f":{port}")
# 比如redis可能传入0
if name is not None:
url_parts.append(f"/{name}")
return "".join(url_parts)
class RedisAsyncClient(aioredis.Redis):
"""异步的客户端
例子::
rdb = RedisAsyncClient()
print(rdb.url)
Args:
host: 服务器地址
port: 服务器端口
user: 用户名
db: 数据库
password: 密码
decode_responses: 字符串输入被编码成utf8存储在Redis里了，而取出来的时候还是被编码后的bytes，需要显示的decode才能变成字符串
health_check_interval: 定时检测连接，防止出现ConnectionErrors (104, Connection reset by peer)
"""
def __init__(
self,
host: str = "localhost",
port: int = 6379,
db: int = 0,
password: str = None,
decode_responses=True,
health_check_interval=10,
socket_connect_timeout=5,
retry_on_timeout=True,
socket_keepalive=True,
**kwargs,
):
super().__init__(
host=host,
port=port,
db=db,
password=password,
decode_responses=decode_responses,
health_check_interval=health_check_interval,
socket_connect_timeout=socket_connect_timeout,
retry_on_timeout=retry_on_timeout,
socket_keepalive=socket_keepalive,
**kwargs,
)
self.url = make_url("redis", host=host, port=port, name=db, password=password)
class RedisCacheInfo(object):
"""统一缓存信息类"""
def __init__(self, key, timeout: Union[int, timedelta] = timedelta(seconds=60), data_type=RedisTypeEnum.String):
"""
缓存信息类初始化
Args:
key: 缓存的key
timeout: 缓存过期时间, 单位秒
data_type: 缓存采用的数据结构 (不传并不影响，用于标记业务采用的是什么数据结构)
"""
self.key = key
self.timeout = timeout
self.data_type = data_type
def __str__(self):
return f"cache key {self.key} timeout {self.timeout}s"
class RedisManager:
client: RedisAsyncClient = None
@classmethod
def init_redis_conn(cls, host, port, password, db):
"""初始化redis 连接"""
if cls.client is None:
cls.client = RedisAsyncClient(host=host, port=port, password=password, db=db)
@classmethod
async def set_with_cache_info(cls, redis_cache_info: RedisCacheInfo, value):
"""
根据 RedisCacheInfo 设置 Redis 缓存
:param redis_cache_info: RedisCacheInfo缓存信息对象
:param value: 缓存的值
:return:
"""
await cls.client.setex(redis_cache_info.key, redis_cache_info.timeout, value)
@classmethod
async def get_with_cache_info(cls, redis_cache_info: RedisCacheInfo):
"""
根据 RedisCacheInfo 获取 Redis 缓存
:param redis_cache_info: RedisCacheInfo 缓存信息对象
:return:
"""
cache_info = await cls.client.get(redis_cache_info.key)
return cache_info
@classmethod
async def del_with_cache_info(cls, redis_cache_info: RedisCacheInfo):
"""
根据 RedisCacheInfo 删除 Redis 缓存
:param redis_cache_info: RedisCacheInfo缓存信息对象
:return:
"""
await cls.client.delete(redis_cache_info.key)
@staticmethod
async def get_or_set_cache(cache_info: RedisCacheInfo, fetch_data_func: Callable[[], Awaitable[dict]]) -> dict:
"""
获取缓存数据，如果缓存不存在，则从提供的函数中获取并设置缓存
当前版本仅支持 json 形式的 string 格式数据
"""
serialized_data = await RedisManager.get_with_cache_info(cache_info)
if serialized_data:
return json.loads(serialized_data)
data = await fetch_data_func()
try:
serialized_data = json.dumps(data)
await RedisManager.set_with_cache_info(cache_info, serialized_data)
except Exception as e:
logger.warning(f"数据 {data} 通过 json 进行序列化缓存失败:{e}")
return data
@classmethod
def is_valid(cls):
return cls.client is not None
class Redis:
def __init__(self, conf: Dict = None):
try:
host = CONFIG.REDIS_HOST
port = int(CONFIG.REDIS_PORT)
pwd = CONFIG.REDIS_PASSWORD
db = CONFIG.REDIS_DB
RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db)
except Exception as e:
logger.warning(f"Redis initialization has failed:{e}")
def is_valid(self):
return RedisManager.is_valid()
async def get(self, key: str) -> str:
if not self.is_valid() or not key:
return None
try:
v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key))
return v
except Exception as e:
logger.exception(f"{e}, stack:{traceback.format_exc()}")
return None
async def set(self, key: str, data: str, timeout_sec: int):
if not self.is_valid() or not key:
return
try:
await RedisManager.set_with_cache_info(
redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data
)
except Exception as e:
logger.exception(f"{e}, stack:{traceback.format_exc()}")

154
metagpt/utils/s3.py Normal file
View file

@ -0,0 +1,154 @@
import base64
import os.path
import traceback
import uuid
from pathlib import Path
from typing import Optional
import aioboto3
import aiofiles
from metagpt.config import CONFIG
from metagpt.const import BASE64_FORMAT
from metagpt.logs import logger
class S3:
"""A class for interacting with Amazon S3 storage."""
def __init__(self):
self.session = aioboto3.Session()
self.auth_config = {
"service_name": "s3",
"aws_access_key_id": CONFIG.S3_ACCESS_KEY,
"aws_secret_access_key": CONFIG.S3_SECRET_KEY,
"endpoint_url": CONFIG.S3_ENDPOINT_URL,
}
async def upload_file(
self,
bucket: str,
local_path: str,
object_name: str,
) -> None:
"""Upload a file from the local path to the specified path of the storage bucket specified in s3.
Args:
bucket: The name of the S3 storage bucket.
local_path: The local file path, including the file name.
object_name: The complete path of the uploaded file to be stored in S3, including the file name.
Raises:
Exception: If an error occurs during the upload process, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
async with aiofiles.open(local_path, mode="rb") as reader:
body = await reader.read()
await client.put_object(Body=body, Bucket=bucket, Key=object_name)
logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.")
except Exception as e:
logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}")
raise e
async def get_object_url(
self,
bucket: str,
object_name: str,
) -> str:
"""Get the URL for a downloadable or preview file stored in the specified S3 bucket.
Args:
bucket: The name of the S3 storage bucket.
object_name: The complete path of the file stored in S3, including the file name.
Returns:
The URL for the downloadable or preview file.
Raises:
Exception: If an error occurs while retrieving the URL, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
file = await client.get_object(Bucket=bucket, Key=object_name)
return str(file["Body"].url)
except Exception as e:
logger.error(f"Failed to get the url for a downloadable or preview file: {e}")
raise e
async def get_object(
self,
bucket: str,
object_name: str,
) -> bytes:
"""Get the binary data of a file stored in the specified S3 bucket.
Args:
bucket: The name of the S3 storage bucket.
object_name: The complete path of the file stored in S3, including the file name.
Returns:
The binary data of the requested file.
Raises:
Exception: If an error occurs while retrieving the file data, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
s3_object = await client.get_object(Bucket=bucket, Key=object_name)
return await s3_object["Body"].read()
except Exception as e:
logger.error(f"Failed to get the binary data of the file: {e}")
raise e
async def download_file(
self, bucket: str, object_name: str, local_path: str, chunk_size: Optional[int] = 128 * 1024
) -> None:
"""Download an S3 object to a local file.
Args:
bucket: The name of the S3 storage bucket.
object_name: The complete path of the file stored in S3, including the file name.
local_path: The local file path where the S3 object will be downloaded.
chunk_size: The size of data chunks to read and write at a time. Default is 128 KB.
Raises:
Exception: If an error occurs during the download process, an exception is raised.
"""
try:
async with self.session.client(**self.auth_config) as client:
s3_object = await client.get_object(Bucket=bucket, Key=object_name)
stream = s3_object["Body"]
async with aiofiles.open(local_path, mode="wb") as writer:
while True:
file_data = await stream.read(chunk_size)
if not file_data:
break
await writer.write(file_data)
except Exception as e:
logger.error(f"Failed to download the file from S3: {e}")
raise e
async def cache(self, data: str, file_ext: str, format: str = "") -> str:
"""Save data to remote S3 and return url"""
object_name = uuid.uuid4().hex + file_ext
path = Path(__file__).parent
pathname = path / object_name
try:
async with aiofiles.open(str(pathname), mode="wb") as file:
if format == BASE64_FORMAT:
data = base64.b64decode(data)
await file.write(data)
bucket = CONFIG.S3_BUCKET
object_pathname = CONFIG.S3_BUCKET or "system"
object_pathname += f"/{object_name}"
object_pathname = os.path.normpath(object_pathname)
await self.upload_file(bucket=bucket, local_path=str(pathname), object_name=object_pathname)
pathname.unlink(missing_ok=True)
return await self.get_object_url(bucket=bucket, object_name=object_pathname)
except Exception as e:
logger.exception(f"{e}, stack:{traceback.format_exc()}")
pathname.unlink(missing_ok=True)
return None