From 9d1a261bf626502a0ac3ee2406a0e7d688c41070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 11:42:13 +0800 Subject: [PATCH 001/378] feat: + education industry --- .gitignore | 3 + metagpt/actions/write_teaching_plan.py | 134 +++++++++++++++++++++++++ metagpt/provider/openai_api.py | 20 ++-- metagpt/roles/role.py | 66 +++++++++++- metagpt/roles/teacher.py | 96 ++++++++++++++++++ metagpt/software_company.py | 5 +- requirements.txt | 1 + startup.py | 96 ++++++++++++++++-- tests/metagpt/roles/test_teacher.py | 94 +++++++++++++++++ 9 files changed, 498 insertions(+), 17 deletions(-) create mode 100644 metagpt/actions/write_teaching_plan.py create mode 100644 metagpt/roles/teacher.py create mode 100644 tests/metagpt/roles/test_teacher.py diff --git a/.gitignore b/.gitignore index c4c79c733..3ec71f8b6 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,6 @@ workspace/* *.mmd tmp output.wav + +# output folder +output diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py new file mode 100644 index 000000000..1f0167df3 --- /dev/null +++ b/metagpt/actions/write_teaching_plan.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 14:43 +@Author : mashenquan +@File : write_teaching_plan.py +""" +from langchain.llms.base import LLM +from metagpt.logs import logger +from metagpt.actions import Action +from metagpt.schema import Message + + +class TeachingPlanRequirement(Action): + """Teaching Plan Requirement without any implementation details""" + + async def run(self, *args, **kwargs): + raise NotImplementedError + + +class WriteTeachingPlanPart(Action): + """Write Teaching Plan Part""" + + def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + """ + + Args: + name: action name + context: context + llm: object of :class:`LLM` + topic: topic part of teaching plan + """ + super().__init__(name, context, llm) + self.topic = topic + self.language = language + self.rsp = None + + async def run(self, *args, **kwargs): + if len(args) < 1 or len(args[0]) < 1 or not isinstance(args[0][0], Message): + raise ValueError("Invalid args, a tuple of List[Message] is expected") + + statements = self.TOPIC_STATEMENTS.get(self.topic, []) + formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE + prompt = formatter.format(formation=self.FORMATION, + role=self.prefix, + statements="\n".join(statements), + lesson=args[0][0].content, + topic=self.topic, + language=self.language) + + logger.debug(prompt) + rsp = await self._aask(prompt=prompt) + logger.debug(rsp) + self._set_result(rsp) + return self.rsp + + def _set_result(self, rsp): + if self.DATA_BEGIN_TAG in rsp: + ix = rsp.index(self.DATA_BEGIN_TAG) + rsp = rsp[ix + len(self.DATA_BEGIN_TAG):] + if self.DATA_END_TAG in rsp: + ix = rsp.index(self.DATA_END_TAG) + rsp = rsp[0:ix] + self.rsp = rsp.strip() + if self.topic != self.COURSE_TITLE: + return + if '#' not in self.rsp or self.rsp.index('#') != 0: + self.rsp = "# " + self.rsp + + def __str__(self): + """str()时返回`topic`""" + return self.topic + + def __repr__(self): + """调试时返回`topic`""" + return self.topic + + FORMATION = """ + "\tCapacity and role" defines the role you are currently playing; + "\t[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook; + "\tStatement" defines the work detail you need to complete at this stage; + "\tAnswer options" defines the format requirements for your responses; + "\tConstraint" defines the conditions that your responses must comply with. + """ + COURSE_TITLE = "Title" + TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", + "Teaching Methods and Strategies", "Learning Activities", + "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement"] + + TOPIC_STATEMENTS = { + COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " + "without anything else."], + "Teaching Content": [ + "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar " + "structures that appear in the textbook, as well as the listening materials and key points.", + "Statement: \"Teaching Content\" must include more examples."], + "Teaching Time Allocation": [ + "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each " + "part of the textbook content."], + "Teaching Methods and Strategies": [ + "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " + "procedures, in detail." + ] + } + + # Teaching plan title + PROMPT_TITLE_TEMPLATE = """ + Do not refer to the context of the previous conversation records, start the conversation anew.\n\n + Formation: {formation}\n\n + {statements}\n + Constraint: Writing in {language}.\n + Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n + [LESSON_BEGIN]\n + {lesson}\n + [LESSON_END] + """ + + # Teaching plan parts: + PROMPT_TEMPLATE = """ + Do not refer to the context of the previous conversation records, start the conversation anew.\n\n + Formation: {formation}\n\n + Capacity and role: {role}\n + Statement: Write the "{topic}" part of teaching plan, WITHOUT ANY content unrelated to "{topic}"!!\n + {statements}\n + Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n + Answer options: Using proper markdown format from second-level header format.\n + Constraint: Writing in {language}.\n + [LESSON_BEGIN]\n + {lesson}\n + [LESSON_END] + """ + + DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]" + DATA_END_TAG = "[TEACHING_PLAN_END]" diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index f6499c643..ba5a655d3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -4,12 +4,13 @@ @Time : 2023/5/5 23:08 @Author : alexanderwu @File : openai.py +@Modified By: mashenquan, 2023-07-27, + try except. """ import asyncio import time from functools import wraps from typing import NamedTuple - +import traceback import openai from metagpt.config import CONFIG @@ -30,7 +31,9 @@ def retry(max_retries): for i in range(max_retries): try: return await f(*args, **kwargs) - except Exception: + except Exception as e: + error_str = traceback.format_exc() + logger.warning(f"Exception occurred: {str(e)}, stack:{error_str}. Retrying...") if i == max_retries - 1: raise await asyncio.sleep(2 ** i) @@ -148,10 +151,15 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await openai.ChatCompletion.acreate( - **self._cons_kwargs(messages), - stream=True - ) + try: + response = await openai.ChatCompletion.acreate( + **self._cons_kwargs(messages), + stream=True + ) + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e # create variables to collect the stream of chunks collected_chunks = [] diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1681586cc..3e18257ed 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,9 +4,11 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py +@Modified By: mashenquan, 2023-07-27, :class:`Role` + properties. """ from __future__ import annotations +import traceback from typing import Iterable, Type from pydantic import BaseModel, Field @@ -92,13 +94,22 @@ class RoleContext(BaseModel): class Role: """角色/代理""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): + def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + # Enable parameter configurability + name = Role.format_value(name, kwargs) + profile = Role.format_value(profile, kwargs) + goal = Role.format_value(goal, kwargs) + constraints = Role.format_value(constraints, kwargs) + desc = Role.format_value(desc, kwargs) + + # Initialize self._llm = LLM() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] self._role_id = str(self._setting) self._rc = RoleContext() + self._options = Role.supply_options(kwargs) def _reset(self): self._states = [] @@ -136,6 +147,26 @@ class Role: """获取角色描述（职位）""" return self._setting.profile + @property + def name(self): + """Return role `name`, read only""" + return self._setting.name + + @property + def desc(self): + """Return role `desc`, read only""" + return self._setting.desc + + @property + def goal(self): + """Return role `goal`, read only""" + return self._setting.goal + + @property + def constraints(self): + """Return role `constraints`, read only""" + return self._setting.constraints + def _get_prefix(self): """获取角色前缀""" if self._setting.desc: @@ -164,7 +195,8 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - response = await self._rc.todo.run(self._rc.important_memory) + requirement = self._rc.important_memory + response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, @@ -238,3 +270,33 @@ class Role: # 将回复发布到环境，等待下一个订阅者处理 self._publish_message(rsp) return rsp + + @staticmethod + def supply_options(options): + """Supply missing options""" + ret = Role.__DEFAULT_OPTIONS__.copy() + if not options: + return ret + ret.update(options) + return ret + + @staticmethod + def format_value(value, options): + """Fill parameters inside `value` with `options`. + """ + if "{" not in value: + return value + + options = Role.supply_options(options) + try: + return value.format(**options) + except KeyError as e: + logger.warning(f"Parameter is missing:{e}") + for k, v in options.items(): + value = value.replace("{" + f"{k}" + "}", v) + return value + + __DEFAULT_OPTIONS__ = { + "teaching_language": "English", + "language": "Chinese" + } \ No newline at end of file diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py new file mode 100644 index 000000000..a007926be --- /dev/null +++ b/metagpt/roles/teacher.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/23 17:25 +@Author : mashenquan +@File : teacher.py +""" +from pathlib import Path + +import aiofiles + +from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement +from metagpt.roles import Role +from metagpt.schema import Message +from metagpt.logs import logger +import re + + +class Teacher(Role): + """Support configurable teacher roles, + with native and teaching languages being replaceable through configurations.""" + def __init__(self, name='Lily', profile='{teaching_language} Teacher', + goal='writing a {language} teaching plan part by part', + constraints='writing in {language}', desc="", *args, **kwargs): + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + actions = [] + for topic in WriteTeachingPlanPart.TOPICS: + act = WriteTeachingPlanPart(topic=topic, llm=self._llm) + actions.append(act) + self._init_actions(actions) + self._watch({TeachingPlanRequirement}) + + async def _think(self) -> None: + """Everything will be done part by part.""" + if self._rc.todo is None: + self._set_state(0) + return + + if self._rc.state + 1 < len(self._states): + self._set_state(self._rc.state + 1) + else: + self._rc.todo = None + + async def _react(self) -> Message: + ret = Message(content="") + while True: + await self._think() + if self._rc.todo is None: + break + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + msg = await self._act() + if ret.content != '': + ret.content += "\n\n\n" + ret.content += msg.content + logger.info(ret.content) + await self.save(ret.content) + return ret + + async def save(self, content): + """Save teaching plan""" + filename = Teacher.new_file_name(self.course_title) + pathname = Path(__file__).resolve().parent.parent.parent / "output" + pathname.mkdir(exist_ok=True) + pathname = pathname / filename + try: + async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + await writer.write(content) + except Exception as e: + logger.error(f'Save failed：{e}') + logger.info(f"Save to:{pathname}") + + @staticmethod + def new_file_name(lesson_title, ext=".md"): + """Create a related file name based on `lesson_title` and `ext`.""" + # 定义需要替换的特殊字符 + illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' + # 将特殊字符替换为下划线 + filename = re.sub(illegal_chars, '_', lesson_title) + ext + return re.sub(r'_+', '_', filename) + + @property + def course_title(self): + """Return course title of teaching plan""" + default_title = "teaching_plan" + for act in self._actions: + if act.topic != WriteTeachingPlanPart.COURSE_TITLE: + continue + if act.rsp is None: + return default_title + title = act.rsp.lstrip("# \n") + if '\n' in title: + ix = title.index('\n') + title = title[0: ix] + return title + + return default_title diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..10fb025d6 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,6 +4,7 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023-07-27, Add `role` & `cause_by` parameters to `start_project()`. """ from pydantic import BaseModel, Field @@ -42,10 +43,10 @@ class SoftwareCompany(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea): + def start_project(self, idea, role="BOSS", cause_by=BossRequirement): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) + self.environment.publish_message(Message(role=role, content=idea, cause_by=cause_by)) def _save(self): logger.info(self.json()) diff --git a/requirements.txt b/requirements.txt index 32a436962..4d5856c20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,3 +35,4 @@ tqdm==4.64.0 anthropic==0.3.6 typing-inspect==0.8.0 typing_extensions==4.5.0 +aiofiles \ No newline at end of file diff --git a/startup.py b/startup.py index e062babb5..17f55fb0a 100644 --- a/startup.py +++ b/startup.py @@ -1,15 +1,23 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023-07-27, +industry concept +""" + import asyncio - +from pathlib import Path +import aiofiles import fire - +from metagpt.logs import logger +from metagpt.actions.write_teaching_plan import TeachingPlanRequirement from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager +from metagpt.roles.teacher import Teacher from metagpt.software_company import SoftwareCompany -async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): - """Run a startup. Be a boss.""" +async def software_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): + """Run a startup. Be a boss in software industry.""" + idea = kwargs['idea'] # Your innovative idea, such as "Creating a snake game." company = SoftwareCompany() company.hire([ProductManager(), Architect(), @@ -20,16 +28,90 @@ async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_rev await company.run(n_round=n_round) -def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): +async def education_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters withthe same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + lesson_file = kwargs.get('lesson_file') + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + company = SoftwareCompany() + company.hire([Teacher(*args, **kwargs)]) + company.invest(investment) + company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + await company.run(n_round=1) + + +def main(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea, such as "Creating a snake game." :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: :param code_review: Whether to use code review. + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py a--param1=value1 --param2=value2` :return: """ - asyncio.run(startup(idea, investment, n_round, code_review)) + industry = kwargs.get("industry", "software") + industries = { + "software": software_startup, + "education": education_startup, + } + startup = industries.get(industry) + if startup is None: + print(f"Available industries:{list(industries.keys())}") + return + asyncio.run(startup(investment, n_round, code_review, *args, **kwargs)) if __name__ == '__main__': diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py new file mode 100644 index 000000000..0dddff3ac --- /dev/null +++ b/tests/metagpt/roles/test_teacher.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/23 17:25 +@Author : mashenquan +@File : teacher.py +""" + +from typing import Dict, Optional +from pydantic import BaseModel + +from metagpt.roles.teacher import Teacher + + +def test_init(): + class Inputs(BaseModel): + name: str + profile: str + goal: str + constraints: str + desc: str + options: Optional[Dict] = None + expect_name: str + expect_profile: str + expect_goal: str + expect_constraints: str + expect_desc: str + + inputs = [ + { + "name": "Lily{language}", + "expect_name": "LilyCN", + "profile": "X {teaching_language}", + "expect_profile": "X EN", + "goal": "Do {something_big}, {language}", + "expect_goal": "Do sleep, CN", + "constraints": "Do in {key1}, {language}", + "expect_constraints": "Do in HaHa, CN", + "options": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, + "desc": "aaa{language}", + "expect_desc": "aaaCN" + }, + { + "name": "Lily{language}", + "expect_name": "LilyChinese", + "profile": "X {teaching_language}", + "expect_profile": "X English", + "goal": "Do {something_big}, {language}", + "expect_goal": "Do {something_big}, Chinese", + "constraints": "Do in {key1}, {language}", + "expect_constraints": "Do in {key1}, Chinese", + "desc": "aaa{language}", + "expect_desc": "aaaChinese" + }, + ] + + for i in inputs: + seed = Inputs(**i) + teacher = Teacher(name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, + desc=seed.desc, options=seed.options) + assert teacher.name == seed.expect_name + assert teacher.desc == seed.expect_desc + assert teacher.profile == seed.expect_profile + assert teacher.goal == seed.expect_goal + assert teacher.constraints == seed.expect_constraints + assert teacher.course_title == "teaching_plan" + + +def test_new_file_name(): + class Inputs(BaseModel): + lesson_title: str + ext: str + expect: str + + inputs = [ + { + "lesson_title": "# @344\n12", + "ext": ".md", + "expect": "_344_12.md" + }, + { + "lesson_title": "1#@$%!*&\\/:*?\"<>|\n\t \'1", + "ext": ".cc", + "expect": "1_1.cc" + } + ] + for i in inputs: + seed = Inputs(**i) + result = Teacher.new_file_name(seed.lesson_title, seed.ext) + assert result == seed.expect + + +if __name__ == '__main__': + test_init() From 5725296b1cdf6f7df494811945e07e4fe797aeb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 14:18:21 +0800 Subject: [PATCH 002/378] fixbug: unit test --- requirements-test.txt | 38 +++++++++++++++++++++++++++++ tests/metagpt/roles/test_teacher.py | 7 +++--- 2 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 requirements-test.txt diff --git a/requirements-test.txt b/requirements-test.txt new file mode 100644 index 000000000..4d5856c20 --- /dev/null +++ b/requirements-test.txt @@ -0,0 +1,38 @@ +aiohttp==3.8.4 +#azure_storage==0.37.0 +channels==4.0.0 +# chromadb==0.3.22 +# Django==4.1.5 +# docx==0.2.4 +duckduckgo_search==2.9.4 +#faiss==1.5.3 +faiss_cpu==1.7.4 +fire==0.4.0 +# godot==0.1.1 +# google_api_python_client==2.93.0 +langchain==0.0.231 +loguru==0.6.0 +meilisearch==0.21.0 +numpy==1.24.3 +openai==0.27.8 +openpyxl +pandas==1.4.1 +pydantic==1.10.7 +#pygame==2.1.3 +#pymilvus==2.2.8 +pytest==7.2.2 +python_docx==0.8.11 +PyYAML==6.0 +# sentence_transformers==2.2.2 +setuptools==65.6.3 +tenacity==8.2.2 +tiktoken==0.3.3 +tqdm==4.64.0 +#unstructured[local-inference] +# playwright +# selenium>4 +# webdriver_manager<3.9 +anthropic==0.3.6 +typing-inspect==0.8.0 +typing_extensions==4.5.0 +aiofiles \ No newline at end of file diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 0dddff3ac..10789f868 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -19,7 +19,7 @@ def test_init(): goal: str constraints: str desc: str - options: Optional[Dict] = None + kwargs: Optional[Dict] = None expect_name: str expect_profile: str expect_goal: str @@ -36,7 +36,7 @@ def test_init(): "expect_goal": "Do sleep, CN", "constraints": "Do in {key1}, {language}", "expect_constraints": "Do in HaHa, CN", - "options": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, + "kwargs": {"language": "CN", "key1": "HaHa", "something_big": "sleep", "teaching_language": "EN"}, "desc": "aaa{language}", "expect_desc": "aaaCN" }, @@ -49,6 +49,7 @@ def test_init(): "expect_goal": "Do {something_big}, Chinese", "constraints": "Do in {key1}, {language}", "expect_constraints": "Do in {key1}, Chinese", + "kwargs": {}, "desc": "aaa{language}", "expect_desc": "aaaChinese" }, @@ -57,7 +58,7 @@ def test_init(): for i in inputs: seed = Inputs(**i) teacher = Teacher(name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, - desc=seed.desc, options=seed.options) + desc=seed.desc, **seed.kwargs) assert teacher.name == seed.expect_name assert teacher.desc == seed.expect_desc assert teacher.profile == seed.expect_profile From 686ca2347817ee913d8e53f51846a4d988cdb899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 15:33:52 +0800 Subject: [PATCH 003/378] fixbug: startup parameters do not match --- requirements-test.txt | 17 ++++++++++------- startup.py | 15 +++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 4d5856c20..7c03dddd9 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,7 +1,7 @@ aiohttp==3.8.4 -#azure_storage==0.37.0 +azure-cognitiveservices-speech==1.30.0 channels==4.0.0 -# chromadb==0.3.22 +chromadb==0.3.22 # Django==4.1.5 # docx==0.2.4 duckduckgo_search==2.9.4 @@ -19,7 +19,7 @@ openpyxl pandas==1.4.1 pydantic==1.10.7 #pygame==2.1.3 -#pymilvus==2.2.8 +pymilvus==2.2.8 pytest==7.2.2 python_docx==0.8.11 PyYAML==6.0 @@ -29,10 +29,13 @@ tenacity==8.2.2 tiktoken==0.3.3 tqdm==4.64.0 #unstructured[local-inference] -# playwright -# selenium>4 -# webdriver_manager<3.9 +playwright +selenium>4 +webdriver_manager<3.9 anthropic==0.3.6 typing-inspect==0.8.0 typing_extensions==4.5.0 -aiofiles \ No newline at end of file +bs4 +aiofiles +pytest +pytest-asyncio \ No newline at end of file diff --git a/startup.py b/startup.py index 17f55fb0a..c05bbbbf0 100644 --- a/startup.py +++ b/startup.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Modified By: mashenquan, 2023-07-27, +industry concept +@Modified By: mashenquan, 2023-07-27, + `industry` concept """ import asyncio @@ -15,9 +15,9 @@ from metagpt.roles.teacher import Teacher from metagpt.software_company import SoftwareCompany -async def software_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): +async def software_startup(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): """Run a startup. Be a boss in software industry.""" - idea = kwargs['idea'] # Your innovative idea, such as "Creating a snake game." + code_review = kwargs.get("code_review", False) # Whether to use code review. company = SoftwareCompany() company.hire([ProductManager(), Architect(), @@ -28,7 +28,7 @@ async def software_startup(investment: float = 3.0, n_round: int = 5, code_revie await company.run(n_round=n_round) -async def education_startup(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): +async def education_startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): """Run a startup. Be a teacher in education industry.""" demo_lesson = """ @@ -76,7 +76,6 @@ async def education_startup(investment: float = 3.0, n_round: int = 5, code_revi """ lesson = "" - lesson_file = kwargs.get('lesson_file') if lesson_file is not None and Path(lesson_file).exists(): async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: lesson = await reader.read() @@ -92,12 +91,12 @@ async def education_startup(investment: float = 3.0, n_round: int = 5, code_revi await company.run(n_round=1) -def main(investment: float = 3.0, n_round: int = 5, code_review: bool = False, *args, **kwargs): +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: - :param code_review: Whether to use code review. :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` :param kwargs: Parameters passed in format: `python your_script.py a--param1=value1 --param2=value2` :return: @@ -111,7 +110,7 @@ def main(investment: float = 3.0, n_round: int = 5, code_review: bool = False, * if startup is None: print(f"Available industries:{list(industries.keys())}") return - asyncio.run(startup(investment, n_round, code_review, *args, **kwargs)) + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) if __name__ == '__main__': From 255c56ca2658a3b2bcf76e8f68edc8b864a32572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:06:32 +0800 Subject: [PATCH 004/378] feat: +test --- .../actions/test_write_teaching_plan.py | 57 +++++++++++++++++++ tests/metagpt/roles/test_teacher.py | 1 + 2 files changed, 58 insertions(+) create mode 100644 tests/metagpt/actions/test_write_teaching_plan.py diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py new file mode 100644 index 000000000..b47d6ab56 --- /dev/null +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -0,0 +1,57 @@ +import asyncio +from typing import Optional +from pydantic import BaseModel +from langchain.llms.base import LLM + +from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart +from metagpt.schema import Message + + +class MockWriteTeachingPlanPart(WriteTeachingPlanPart): + def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + super().__init__(name, context, llm, topic, language) + + async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str: + return f"{WriteTeachingPlanPart.DATA_BEGIN_TAG}\nprompt\n{WriteTeachingPlanPart.DATA_END_TAG}" + + +async def mock_write_teaching_plan_part(): + class Inputs(BaseModel): + input: str + name: str + topic: str + language: str + + inputs = [ + { + "input": "AABBCC", + "name": "A", + "topic": "B", + "language": "C" + }, + { + "input": "DDEEFFF", + "name": "A1", + "topic": "B1", + "language": "C1" + } + ] + + for i in inputs: + seed = Inputs(**i) + act = MockWriteTeachingPlanPart(name=seed.name, topic=seed.topic, language=seed.language) + await act.run([Message(content="")]) + assert act.topic == seed.topic + assert str(act) == seed.topic + assert act.name == seed.name + assert act.rsp == "prompt" + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_write_teaching_plan_part()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 10789f868..3af053338 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -93,3 +93,4 @@ def test_new_file_name(): if __name__ == '__main__': test_init() + test_new_file_name() From 2e6f88d3e4ab556f79626bf91367de7cd945de72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:10:54 +0800 Subject: [PATCH 005/378] feat: + notation --- metagpt/actions/write_teaching_plan.py | 2 +- metagpt/roles/teacher.py | 2 +- tests/metagpt/actions/test_write_teaching_plan.py | 8 ++++++++ tests/metagpt/roles/test_teacher.py | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 1f0167df3..76c72651d 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/11 14:43 +@Time : 2023/7/27 @Author : mashenquan @File : write_teaching_plan.py """ diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index a007926be..acaa3860f 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/23 17:25 +@Time : 2023/7/27 @Author : mashenquan @File : teacher.py """ diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index b47d6ab56..2e34491fb 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -1,3 +1,11 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/7/28 17:25 +@Author : mashenquan +@File : test_write_teaching_plan.py +""" + import asyncio from typing import Optional from pydantic import BaseModel diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 3af053338..5faa43455 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -1,9 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/5/23 17:25 +@Time : 2023/7/27 13:25 @Author : mashenquan -@File : teacher.py +@File : test_teacher.py """ from typing import Dict, Optional From 34e5658009016cefd0083c0797ef6d3f42b68fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:14:37 +0800 Subject: [PATCH 006/378] feat: + notation --- metagpt/actions/write_teaching_plan.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 76c72651d..0778b86b4 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -68,11 +68,11 @@ class WriteTeachingPlanPart(Action): self.rsp = "# " + self.rsp def __str__(self): - """str()时返回`topic`""" + """Return `topic` value when str()""" return self.topic def __repr__(self): - """调试时返回`topic`""" + """Show `topic` value when debug""" return self.topic FORMATION = """ From d7c1d9797f82d80015f7ec3e548466c9b25b938c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:22:28 +0800 Subject: [PATCH 007/378] fixbug: prompt format error --- metagpt/actions/write_teaching_plan.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 0778b86b4..370c70040 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -76,11 +76,11 @@ class WriteTeachingPlanPart(Action): return self.topic FORMATION = """ - "\tCapacity and role" defines the role you are currently playing; - "\t[LESSON_BEGIN]" and "[LESSON_END]" tags enclose the content of textbook; - "\tStatement" defines the work detail you need to complete at this stage; - "\tAnswer options" defines the format requirements for your responses; - "\tConstraint" defines the conditions that your responses must comply with. + \t\"Capacity and role\" defines the role you are currently playing; + \t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook; + \t\"Statement\" defines the work detail you need to complete at this stage; + \t\"Answer options\" defines the format requirements for your responses; + \t\"Constraint\" defines the conditions that your responses must comply with. """ COURSE_TITLE = "Title" TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", From d79fe56a38596385a6ea5aafdb5752942bbebedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:34:30 +0800 Subject: [PATCH 008/378] fixbug: prompt format error --- metagpt/actions/write_teaching_plan.py | 60 +++++++++---------- .../actions/test_write_teaching_plan.py | 4 +- 2 files changed, 32 insertions(+), 32 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 370c70040..4c36983ff 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -75,13 +75,12 @@ class WriteTeachingPlanPart(Action): """Show `topic` value when debug""" return self.topic - FORMATION = """ - \t\"Capacity and role\" defines the role you are currently playing; - \t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook; - \t\"Statement\" defines the work detail you need to complete at this stage; - \t\"Answer options\" defines the format requirements for your responses; - \t\"Constraint\" defines the conditions that your responses must comply with. - """ + FORMATION = "\"Capacity and role\" defines the role you are currently playing;\n" \ + "\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n" \ + "\t\"Statement\" defines the work detail you need to complete at this stage;\n" \ + "\t\"Answer options\" defines the format requirements for your responses;\n" \ + "\t\"Constraint\" defines the conditions that your responses must comply with." + COURSE_TITLE = "Title" TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", @@ -104,31 +103,32 @@ class WriteTeachingPlanPart(Action): } # Teaching plan title - PROMPT_TITLE_TEMPLATE = """ - Do not refer to the context of the previous conversation records, start the conversation anew.\n\n - Formation: {formation}\n\n - {statements}\n - Constraint: Writing in {language}.\n - Answer options: Encloses the lesson title with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n - [LESSON_BEGIN]\n - {lesson}\n - [LESSON_END] - """ + PROMPT_TITLE_TEMPLATE = "Do not refer to the context of the previous conversation records, " \ + "start the conversation anew.\n\n" \ + "Formation: {formation}\n\n" \ + "{statements}\n" \ + "Constraint: Writing in {language}.\n" \ + "Answer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" " \ + "and \"[TEACHING_PLAN_END]\" tags.\n" \ + "[LESSON_BEGIN]\n" \ + "{lesson}\n" \ + "[LESSON_END]" # Teaching plan parts: - PROMPT_TEMPLATE = """ - Do not refer to the context of the previous conversation records, start the conversation anew.\n\n - Formation: {formation}\n\n - Capacity and role: {role}\n - Statement: Write the "{topic}" part of teaching plan, WITHOUT ANY content unrelated to "{topic}"!!\n - {statements}\n - Answer options: Enclose the teaching plan content with "[TEACHING_PLAN_BEGIN]" and "[TEACHING_PLAN_END]" tags.\n - Answer options: Using proper markdown format from second-level header format.\n - Constraint: Writing in {language}.\n - [LESSON_BEGIN]\n - {lesson}\n - [LESSON_END] - """ + PROMPT_TEMPLATE = "Do not refer to the context of the previous conversation records, " \ + "start the conversation anew.\n\n" \ + "Formation: {formation}\n\n" \ + "Capacity and role: {role}\n" \ + "Statement: Write the \"{topic}\" part of teaching plan, " \ + "WITHOUT ANY content unrelated to \"{topic}\"!!\n" \ + "{statements}\n" \ + "Answer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" " \ + "and \"[TEACHING_PLAN_END]\" tags.\n" \ + "Answer options: Using proper markdown format from second-level header format.\n" \ + "Constraint: Writing in {language}.\n" \ + "[LESSON_BEGIN]\n" \ + "{lesson}\n" \ + "[LESSON_END]" DATA_BEGIN_TAG = "[TEACHING_PLAN_BEGIN]" DATA_END_TAG = "[TEACHING_PLAN_END]" diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index 2e34491fb..299a89639 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -34,7 +34,7 @@ async def mock_write_teaching_plan_part(): { "input": "AABBCC", "name": "A", - "topic": "B", + "topic": WriteTeachingPlanPart.COURSE_TITLE, "language": "C" }, { @@ -52,7 +52,7 @@ async def mock_write_teaching_plan_part(): assert act.topic == seed.topic assert str(act) == seed.topic assert act.name == seed.name - assert act.rsp == "prompt" + assert act.rsp == "# prompt" if seed.topic == WriteTeachingPlanPart.COURSE_TITLE else "prompt" def test_suite(): From b95a01079c4a0cc18119ef196f5f0124556fbdc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:42:02 +0800 Subject: [PATCH 009/378] feat: + notation --- metagpt/roles/teacher.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index acaa3860f..fede9f74a 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -72,9 +72,9 @@ class Teacher(Role): @staticmethod def new_file_name(lesson_title, ext=".md"): """Create a related file name based on `lesson_title` and `ext`.""" - # 定义需要替换的特殊字符 + # Define the special characters that need to be replaced. illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' - # 将特殊字符替换为下划线 + # Replace the special characters with underscores. filename = re.sub(illegal_chars, '_', lesson_title) + ext return re.sub(r'_+', '_', filename) From 16bad64649cbdc3b498376981dd0039a63e0eff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 28 Jul 2023 16:44:30 +0800 Subject: [PATCH 010/378] feat: + notation --- startup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/startup.py b/startup.py index c05bbbbf0..1fe4a067a 100644 --- a/startup.py +++ b/startup.py @@ -98,7 +98,7 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py a--param1=value1 --param2=value2` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` :return: """ industry = kwargs.get("industry", "software") From 7cab03942ebdc9a7fbfcc34f31d0bf8d780d6fb3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 15:44:12 +0800 Subject: [PATCH 011/378] =?UTF-8?q?feat:=20+=E7=BB=83=E4=B9=A0=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/write_teaching_plan.py | 57 +++++++++++++++++++++++++- metagpt/roles/role.py | 2 +- startup.py | 2 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 4c36983ff..66c370bc9 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -39,7 +39,12 @@ class WriteTeachingPlanPart(Action): if len(args) < 1 or len(args[0]) < 1 or not isinstance(args[0][0], Message): raise ValueError("Invalid args, a tuple of List[Message] is expected") - statements = self.TOPIC_STATEMENTS.get(self.topic, []) + statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, []) + statements = [] + from metagpt.roles import Role + for p in statement_patterns: + s = Role.format_value(p, kwargs) + statements.append(s) formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE prompt = formatter.format(formation=self.FORMATION, role=self.prefix, @@ -84,7 +89,9 @@ class WriteTeachingPlanPart(Action): COURSE_TITLE = "Title" TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", - "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement"] + "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", + "Vocabulary Practice", "Grammar Practice", "Reading Comprehension", "Listening Practice", + "Writing Practice", "Speaking Practice", "Translation Practice", "Listening and Speaking Activities"] TOPIC_STATEMENTS = { COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " @@ -99,6 +106,52 @@ class WriteTeachingPlanPart(Action): "Teaching Methods and Strategies": [ "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " "procedures, in detail." + ], + "Vocabulary Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create vocabulary practice exercises. The exercises should be in either {language} with " + "{teaching_language} answers or {teaching_language} with {language} answers. The key-related vocabulary " + "and phrases in the textbook content must all be included in the exercises." + ], + "Grammar Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create grammar practice exercises. "], + "Reading Comprehension": [ + "Statement: Based on the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create {teaching_language} reading comprehension exercises. ", + "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " + "and \"[LESSON_END]\" tags.", + "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Answer options: Write the story content in {teaching_language}." + ], + "Listening Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create listening practice exercises. Each exercise should include the audio content and the " + "question-and-answer part." + ], + "Writing Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create writing practice exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + ], + "Speaking Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create speaking practice exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + ], + "Translation Practice": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create Translation practice exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + ], + "Listening and Speaking Activities": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create listening and speaking activities exercises.", + #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ] } diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 3e18257ed..47aa90197 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -196,7 +196,7 @@ class Role: logger.info(f"{self._setting}: ready to {self._rc.todo}") requirement = self._rc.important_memory - response = await self._rc.todo.run(requirement) + response = await self._rc.todo.run(requirement, **self._options) # logger.info(response) if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, diff --git a/startup.py b/startup.py index 1fe4a067a..ee8cd3b6e 100644 --- a/startup.py +++ b/startup.py @@ -69,7 +69,7 @@ async def education_startup(lesson_file: str, investment: float = 3.0, n_round: 3a Listen, say and trace Aa Bb Cc Dd Ee Ff Gg - 3b Listen and number the following letters. Then circle the letters withthe same sound as Bb. + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. Aa Bb Cc Dd Ee Ff Gg 3c Match the big letters with the small ones. Then write them on the lines. From 8b7eddad86c47e76b200c3ed4cbe1c2377a3767f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 15:57:06 +0800 Subject: [PATCH 012/378] =?UTF-8?q?feat:=20+=E7=BB=83=E4=B9=A0=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/write_teaching_plan.py | 33 +++++++++++++------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 66c370bc9..09b45634c 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -87,7 +87,8 @@ class WriteTeachingPlanPart(Action): "\t\"Constraint\" defines the conditions that your responses must comply with." COURSE_TITLE = "Title" - TOPICS = [COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", + TOPICS = [ + COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", "Vocabulary Practice", "Grammar Practice", "Reading Comprehension", "Listening Practice", @@ -117,41 +118,41 @@ class WriteTeachingPlanPart(Action): "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create grammar practice exercises. "], "Reading Comprehension": [ - "Statement: Based on the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create {teaching_language} reading comprehension exercises. ", - "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " - "and \"[LESSON_END]\" tags.", - "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Answer options: Write the story content in {teaching_language}." + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create {teaching_language} reading comprehension exercises. " + # "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " + # "and \"[LESSON_END]\" tags.", + # "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", + # "Answer options: Write the story content in {teaching_language}." ], "Listening Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create listening practice exercises. Each exercise should include the audio content and the " "question-and-answer part." ], "Writing Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create writing practice exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ], "Speaking Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create speaking practice exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ], "Translation Practice": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create Translation practice exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ], "Listening and Speaking Activities": [ - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " "create listening and speaking activities exercises.", #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - "Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." + #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." ] } From 444b609e38c931fa18fcf04d35b7ed4016fb46d6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 17:27:09 +0800 Subject: [PATCH 013/378] =?UTF-8?q?feat:=20=E5=88=A0=E6=8E=89=E6=97=A0?= =?UTF-8?q?=E7=94=A8=E7=9A=84part?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- examples/write_teaching_plan.py | 94 ++++++++++++++++++++++++++ metagpt/actions/write_teaching_plan.py | 53 +++------------ 2 files changed, 102 insertions(+), 45 deletions(-) create mode 100644 examples/write_teaching_plan.py diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py new file mode 100644 index 000000000..ec8ad8948 --- /dev/null +++ b/examples/write_teaching_plan.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023-07-27, + `industry` concept +""" + +import asyncio +from pathlib import Path +import aiofiles +import fire +from metagpt.logs import logger +from metagpt.actions.write_teaching_plan import TeachingPlanRequirement +from metagpt.roles.teacher import Teacher +from metagpt.software_company import SoftwareCompany + + +async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + company = SoftwareCompany() + company.hire([Teacher(*args, **kwargs)]) + company.invest(investment) + company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + await company.run(n_round=1) + + +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): + """ + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. + :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + :param n_round: + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :return: + """ + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + + +if __name__ == '__main__': + fire.Fire(main) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 09b45634c..bd6e96956 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -89,10 +89,10 @@ class WriteTeachingPlanPart(Action): COURSE_TITLE = "Title" TOPICS = [ COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", - "Teaching Methods and Strategies", "Learning Activities", - "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", - "Vocabulary Practice", "Grammar Practice", "Reading Comprehension", "Listening Practice", - "Writing Practice", "Speaking Practice", "Translation Practice", "Listening and Speaking Activities"] + "Teaching Methods and Strategies", "Learning Activities", + "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", + "Vocabulary Cloze", "Grammar Questions" + ] TOPIC_STATEMENTS = { COURSE_TITLE: ["Statement: Find and return the title of the lesson only in markdown first-level header format, " @@ -108,52 +108,15 @@ class WriteTeachingPlanPart(Action): "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, " "procedures, in detail." ], - "Vocabulary Practice": [ + "Vocabulary Cloze": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create vocabulary practice exercises. The exercises should be in either {language} with " + "create vocabulary cloze. The cloze should be in either {language} with " "{teaching_language} answers or {teaching_language} with {language} answers. The key-related vocabulary " "and phrases in the textbook content must all be included in the exercises." ], - "Grammar Practice": [ + "Grammar Questions": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create grammar practice exercises. "], - "Reading Comprehension": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create {teaching_language} reading comprehension exercises. " - # "Statement: Prohibit the use of words that are not within the scope of the \"[LESSON_BEGIN]\" " - # "and \"[LESSON_END]\" tags.", - # "Statement: Prohibit copy the content of the \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - # "Answer options: Write the story content in {teaching_language}." - ], - "Listening Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create listening practice exercises. Each exercise should include the audio content and the " - "question-and-answer part." - ], - "Writing Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create writing practice exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ], - "Speaking Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create speaking practice exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ], - "Translation Practice": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create Translation practice exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ], - "Listening and Speaking Activities": [ - "Statement: Using the vocabulary of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create listening and speaking activities exercises.", - #"Statement: Prohibit using content not related to \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags.", - #"Statement: Prohibit copying the content enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags." - ] + "create grammar questions. "] } # Teaching plan title From 3b0f76a1cd1ffada96f5e47ab6c91e7618b5efaf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 17:48:06 +0800 Subject: [PATCH 014/378] =?UTF-8?q?fixbug:=20=E7=BB=9F=E4=B8=80=E4=BA=86?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/design_api.py | 3 ++- metagpt/actions/project_management.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..55213a4b0 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -135,7 +135,8 @@ class WriteDesign(Action): self._save_prd(docs_path, resources_path, context[-1].content) self._save_system_design(docs_path, resources_path, content) - async def run(self, context): + async def run(self, *args, **kwargs): + context = args[0] prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..394f279e8 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -115,7 +115,8 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - async def run(self, context): + async def run(self, *args, **kwargs): + context = args[0] prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) self._save(context, rsp) From 4379d360228e6ded6900ca855658daca02c80172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 31 Jul 2023 17:48:41 +0800 Subject: [PATCH 015/378] =?UTF-8?q?fixbug:=20=E7=BB=9F=E4=B8=80=E4=BA=86?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- startup.py | 97 +++++------------------------------------------------- 1 file changed, 8 insertions(+), 89 deletions(-) diff --git a/startup.py b/startup.py index ee8cd3b6e..e062babb5 100644 --- a/startup.py +++ b/startup.py @@ -1,23 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Modified By: mashenquan, 2023-07-27, + `industry` concept -""" - import asyncio -from pathlib import Path -import aiofiles + import fire -from metagpt.logs import logger -from metagpt.actions.write_teaching_plan import TeachingPlanRequirement + from metagpt.roles import Architect, Engineer, ProductManager, ProjectManager -from metagpt.roles.teacher import Teacher from metagpt.software_company import SoftwareCompany -async def software_startup(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): - """Run a startup. Be a boss in software industry.""" - code_review = kwargs.get("code_review", False) # Whether to use code review. +async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): + """Run a startup. Be a boss.""" company = SoftwareCompany() company.hire([ProductManager(), Architect(), @@ -28,89 +20,16 @@ async def software_startup(idea: str, investment: float = 3.0, n_round: int = 5, await company.run(n_round=n_round) -async def education_startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): - """Run a startup. Be a teacher in education industry.""" - - demo_lesson = """ - UNIT 1 Making New Friends - TOPIC 1 Welcome to China! - Section A - - 1a Listen and number the following names. - Jane Mari Kangkang Michael - Look, listen and understand. Then practice the conversation. - Work in groups. Introduce yourself using - I ’m ... Then practice 1a - with your own hometown or the following places. - - 1b Listen and number the following names - Jane Michael Maria Kangkang - 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. - China the USA the UK Hong Kong Beijing - - 2a Look, listen and understand. Then practice the conversation - Hello! - Hello! - Hello! - Hello! Are you Maria? - No, I’m not. I’m Jane. - Oh, nice to meet you, Jane - Nice to meet you, too. - Hi, Maria! - Hi, Kangkang! - Welcome to China! - Thanks. - - 2b Work in groups. Make up a conversation with your own name and the - following structures. - A: Hello! / Good morning! / Hi! I’m ... Are you ... ? - B: ... - - 3a Listen, say and trace - Aa Bb Cc Dd Ee Ff Gg - - 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. - Aa Bb Cc Dd Ee Ff Gg - - 3c Match the big letters with the small ones. Then write them on the lines. - """ - - lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): - async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: - lesson = await reader.read() - logger.info(f"Course content: {lesson}") - if not lesson: - logger.info("No course content provided, using the demo course.") - lesson = demo_lesson - - company = SoftwareCompany() - company.hire([Teacher(*args, **kwargs)]) - company.invest(investment) - company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) - await company.run(n_round=1) - - -def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): +def main(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. + :param idea: Your innovative idea, such as "Creating a snake game." :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. :param n_round: - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :param code_review: Whether to use code review. :return: """ - industry = kwargs.get("industry", "software") - industries = { - "software": software_startup, - "education": education_startup, - } - startup = industries.get(industry) - if startup is None: - print(f"Available industries:{list(industries.keys())}") - return - asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + asyncio.run(startup(idea, investment, n_round, code_review)) if __name__ == '__main__': From 85c7148b6235fb91a8d141b1d6be990e8999ced8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 10:04:21 +0800 Subject: [PATCH 016/378] feat: +param type --- metagpt/actions/write_teaching_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index bd6e96956..10fc2863f 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -21,7 +21,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + def __init__(self, name: str = "", context=None, llm: LLM = None, topic: str = "", language: str = "Chinese"): """ Args: From d415ca5dbc5a9622cb65b331d7ca87c224de57a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 10:48:26 +0800 Subject: [PATCH 017/378] fixbug: tests --- metagpt/actions/action.py | 1 + metagpt/actions/design_api.py | 3 +-- metagpt/actions/project_management.py | 3 +-- metagpt/actions/write_teaching_plan.py | 9 ++++----- metagpt/roles/teacher.py | 3 ++- tests/metagpt/actions/test_ui_design.py | 10 ++++------ tests/metagpt/actions/test_write_code.py | 3 ++- 7 files changed, 15 insertions(+), 17 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index fa0d592a3..6b9ea626c 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -15,6 +15,7 @@ from metagpt.llm import LLM from metagpt.utils.common import OutputParser from metagpt.logs import logger + class Action(ABC): def __init__(self, name: str = '', context=None, llm: LLM = None): self.name: str = name diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 55213a4b0..1447eacc3 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -135,8 +135,7 @@ class WriteDesign(Action): self._save_prd(docs_path, resources_path, context[-1].content) self._save_system_design(docs_path, resources_path, content) - async def run(self, *args, **kwargs): - context = args[0] + async def run(self, context): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 394f279e8..89c59dcda 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -115,8 +115,7 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - async def run(self, *args, **kwargs): - context = args[0] + async def run(self, context): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) self._save(context, rsp) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 10fc2863f..e8fe110d8 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -5,7 +5,6 @@ @Author : mashenquan @File : write_teaching_plan.py """ -from langchain.llms.base import LLM from metagpt.logs import logger from metagpt.actions import Action from metagpt.schema import Message @@ -21,7 +20,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, name: str = "", context=None, llm: LLM = None, topic: str = "", language: str = "Chinese"): + def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ Args: @@ -35,8 +34,8 @@ class WriteTeachingPlanPart(Action): self.language = language self.rsp = None - async def run(self, *args, **kwargs): - if len(args) < 1 or len(args[0]) < 1 or not isinstance(args[0][0], Message): + async def run(self, messages, *args, **kwargs): + if len(messages) < 1 or not isinstance(messages[0], Message): raise ValueError("Invalid args, a tuple of List[Message] is expected") statement_patterns = self.TOPIC_STATEMENTS.get(self.topic, []) @@ -49,7 +48,7 @@ class WriteTeachingPlanPart(Action): prompt = formatter.format(formation=self.FORMATION, role=self.prefix, statements="\n".join(statements), - lesson=args[0][0].content, + lesson=messages[0].content, topic=self.topic, language=self.language) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index fede9f74a..5d10c4d17 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -10,6 +10,7 @@ from pathlib import Path import aiofiles from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement +from metagpt.const import WORKSPACE_ROOT from metagpt.roles import Role from metagpt.schema import Message from metagpt.logs import logger @@ -59,7 +60,7 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = Path(__file__).resolve().parent.parent.parent / "output" + pathname = WORKSPACE_ROOT / "output" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: diff --git a/tests/metagpt/actions/test_ui_design.py b/tests/metagpt/actions/test_ui_design.py index d284b20f2..dedd0b30e 100644 --- a/tests/metagpt/actions/test_ui_design.py +++ b/tests/metagpt/actions/test_ui_design.py @@ -4,7 +4,7 @@ # from tests.metagpt.roles.ui_role import UIDesign -llm_resp= ''' +llm_resp = ''' # UI Design Description ```The user interface for the snake game will be designed in a way that is simple, clean, and intuitive. The main elements of the game such as the game grid, snake, food, score, and game over message will be clearly defined and easy to understand. The game grid will be centered on the screen with the score displayed at the top. The game controls will be intuitive and easy to use. The design will be modern and minimalist with a pleasing color scheme.``` @@ -100,6 +100,7 @@ body { font-size: 3em; ''' + def test_ui_design_parse_css(): ui_design_work = UIDesign(name="UI design action") @@ -161,7 +162,7 @@ def test_ui_design_parse_css(): transform: translate(-50%, -50%); font-size: 3em; ''' - assert ui_design_work.parse_css_code(context=llm_resp)==css + assert ui_design_work.parse_css_code(context=llm_resp) == css def test_ui_design_parse_html(): @@ -185,7 +186,4 @@ def test_ui_design_parse_html(): ''' - assert ui_design_work.parse_css_code(context=llm_resp)==html - - - + assert ui_design_work.parse_css_code(context=llm_resp) == html diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 7bb18ddf2..2d4c496e1 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modified By: mashenquan, 2023-8-1, fix-bug: `filename` of `write_code.run()` is missing. """ import pytest @@ -18,7 +19,7 @@ async def test_write_code(): api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" write_code = WriteCode("write_code") - code = await write_code.run(api_design) + code = await write_code.run(context=api_design, filename="test") logger.info(code) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 From a56e9a29e3243d6584c326a171e19dc38bf2f906 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 10:53:34 +0800 Subject: [PATCH 018/378] feat: change save to --- metagpt/roles/teacher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 5d10c4d17..95d54133b 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -60,7 +60,7 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = WORKSPACE_ROOT / "output" + pathname = WORKSPACE_ROOT / "teaching_plan" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: From 8f5b3e076e9655f0dde939dd9ed6e3a9e49ac27c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 11:52:50 +0800 Subject: [PATCH 019/378] feat: +Choice Questions, Translation Questions --- metagpt/actions/write_teaching_plan.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index e8fe110d8..2916d7309 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -90,7 +90,7 @@ class WriteTeachingPlanPart(Action): COURSE_TITLE, "Teaching Hours", "Teaching Objectives", "Teaching Content", "Teaching Methods and Strategies", "Learning Activities", "Teaching Time Allocation", "Assessment and Feedback", "Teaching Summary and Improvement", - "Vocabulary Cloze", "Grammar Questions" + "Vocabulary Cloze", "Choice Questions", "Grammar Questions", "Translation Questions" ] TOPIC_STATEMENTS = { @@ -109,13 +109,20 @@ class WriteTeachingPlanPart(Action): ], "Vocabulary Cloze": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create vocabulary cloze. The cloze should be in either {language} with " - "{teaching_language} answers or {teaching_language} with {language} answers. The key-related vocabulary " - "and phrases in the textbook content must all be included in the exercises." + "create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} " + "answers, and it should also include 10 {teaching_language} questions with {language} answers. " + "The key-related vocabulary and phrases in the textbook content must all be included in the exercises.", ], "Grammar Questions": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create grammar questions. "] + "create grammar questions. 10 questions."], + "Choice Questions": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create choice questions. 10 questions."], + "Translation Questions": [ + "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " + "create translation questions. 10 questions." + ] } # Teaching plan title From 0c1febfc77555df93051f00e245d38d626028b79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 12:06:03 +0800 Subject: [PATCH 020/378] feat: +Choice Questions, Translation Questions --- metagpt/actions/write_teaching_plan.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 2916d7309..1c9b1a86e 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -121,7 +121,9 @@ class WriteTeachingPlanPart(Action): "create choice questions. 10 questions."], "Translation Questions": [ "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", " - "create translation questions. 10 questions." + "create translation questions. The translation should include 10 {language} questions with " + "{teaching_language} answers, and it should also include 10 {teaching_language} questions with " + "{language} answers." ] } From e5885ec99ae2ab3fdb0d459d8572bfe03b646e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 12:07:58 +0800 Subject: [PATCH 021/378] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9=E6=A0=BC?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/roles/role.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 47aa90197..36dfb2d36 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -282,8 +282,7 @@ class Role: @staticmethod def format_value(value, options): - """Fill parameters inside `value` with `options`. - """ + """Fill parameters inside `value` with `options`.""" if "{" not in value: return value From a4017a1eeca186ac41cec05a851f200fab8c33d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 13:08:17 +0800 Subject: [PATCH 022/378] feat: + annotation --- examples/write_teaching_plan.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index ec8ad8948..da97a5463 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -80,9 +80,9 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): """ We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: Your innovative idea for `software` industry, such as "Creating a snake game."; lesson filename for `education` industry. + :param idea: lesson filename. :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - :param n_round: + :param n_round: Reserved. :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` :return: @@ -91,4 +91,11 @@ def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): if __name__ == '__main__': + """ + Formats: + ``` + python write_teaching_plan.py lesson_filename --teaching_language= --language= + ``` + If `lesson_filename` is not available, a demo lesson content will be used. + """ fire.Fire(main) From b8901f2bb17cde7eb4cff9e85e1269d0664ec1ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 1 Aug 2023 13:23:17 +0800 Subject: [PATCH 023/378] feat: +annotation --- metagpt/actions/write_teaching_plan.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 1c9b1a86e..3718c9801 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -23,11 +23,11 @@ class WriteTeachingPlanPart(Action): def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ - Args: - name: action name - context: context - llm: object of :class:`LLM` - topic: topic part of teaching plan + :param name: action name + :param context: context + :param llm: object of :class:`LLM` + :param topic: topic part of teaching plan + :param language: A human language, such as Chinese, English, French, etc. """ super().__init__(name, context, llm) self.topic = topic From 80a189ad4a1546f8c1a9dbe00c42725868c35e5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 7 Aug 2023 11:24:34 +0800 Subject: [PATCH 024/378] feat: +meta role --- .gitignore | 1 + config/pattern/write_teaching_plan.yaml | 51 +++++++++++++ examples/write_teaching_plan.py | 5 +- metagpt/roles/fork_meta_role.py | 98 +++++++++++++++++++++++++ metagpt/roles/meta_role.py | 29 ++++++++ 5 files changed, 183 insertions(+), 1 deletion(-) create mode 100644 config/pattern/write_teaching_plan.yaml create mode 100644 metagpt/roles/fork_meta_role.py create mode 100644 metagpt/roles/meta_role.py diff --git a/.gitignore b/.gitignore index 3ec71f8b6..e326e8372 100644 --- a/.gitignore +++ b/.gitignore @@ -163,6 +163,7 @@ workspace/* *.mmd tmp output.wav +*.bak # output folder output diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml new file mode 100644 index 000000000..fbd8b4ae9 --- /dev/null +++ b/config/pattern/write_teaching_plan.yaml @@ -0,0 +1,51 @@ +# `fork` role demo +- role_type: "fork" + name: "Lily" + profile: "{teaching_language} Teacher" + goal: "writing a {language} teaching plan part by part" + constraints: "writing in {language}" + desc: "" + actions: + - name: "" + topic: "Title" + language: "Chinese" + statements: + - "Statement: Find and return the title of the lesson only in markdown first-level header format, without anything else." + - name: "" + topic: "Teaching Content" + language: "Chinese" + statements: + - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar structures that appear in the textbook, as well as the listening materials and key points." + - "Statement: \"Teaching Content\" must include more examples." + - name: "" + topic: "Teaching Time Allocation" + language: "Chinese" + statements: + - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." + - name: "" + topic: "Teaching Methods and Strategies" + language: "Chinese" + statements: + - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, procedures, in detail." + - name: "" + topic: "Vocabulary Cloze" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers. The key-related vocabulary and phrases in the textbook content must all be included in the exercises." + - name: "" + topic: "Grammar Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." + - name: "" + topic: "Choice Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create choice questions. 10 questions." + - name: "" + topic: "Translation Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create translation questions. The translation should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers." + + diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index da97a5463..30a8d8366 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -1,7 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Modified By: mashenquan, 2023-07-27, + `industry` concept +@Time : 2023-07-27 +@Author : mashenquan +@File : write_teaching_plan.py +@Desc: Write teaching plan demo """ import asyncio diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py new file mode 100644 index 000000000..1a69b9ca7 --- /dev/null +++ b/metagpt/roles/fork_meta_role.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : fork_meta_role.py +@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。这是一个`fork` meta-role demo，实现的是write_teaching_plan功能。 +""" + + + +async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + + + company = SoftwareCompany() + company.hire([(*args, **kwargs)]) + company.invest(investment) + company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + await company.run(n_round=1) + + +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): + """ + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: lesson filename. + :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + :param n_round: Reserved. + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :return: + """ + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + + +if __name__ == '__main__': + """ + Formats: + ``` + python write_teaching_plan.py lesson_filename --teaching_language= --language= + ``` + If `lesson_filename` is not available, a demo lesson content will be used. + """ + fire.Fire(main) diff --git a/metagpt/roles/meta_role.py b/metagpt/roles/meta_role.py new file mode 100644 index 000000000..1da180355 --- /dev/null +++ b/metagpt/roles/meta_role.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : meta_role.py +@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。 + 分工参照UML 2.0 activity diagrams: `https://www.uml-diagrams.org/activity-diagrams.html` +""" +from typing import Dict, List + +from metagpt.roles import Role +from pydantic import BaseModel + +class UMLMetaRoleArgs(BaseModel): + role_type: str + name: str = "" + profile: str = "" + goal: str = "" + constraints: str = "" + desc: str = "" + actions: List + +class UMLMetaRole(Role): + """UML activity roles抽象父类""" + + def __init__(self, role_args: Dict): + """""" + self.role_args From 5702aaa5ad4e9113e954ab16e1fde1965b3f591b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 7 Aug 2023 21:12:27 +0800 Subject: [PATCH 025/378] feat: + uml fork style role demo --- config/pattern/template.yaml | 40 +++++ config/pattern/write_teaching_plan.yaml | 103 +++++++++++-- examples/fork_meta_role.py | 121 +++++++++++++++ metagpt/actions/meta_action.py | 61 ++++++++ metagpt/memory/memory.py | 10 +- metagpt/roles/fork_meta_role.py | 187 ++++++++++++++---------- metagpt/roles/meta_role.py | 29 ---- metagpt/roles/role.py | 6 +- metagpt/roles/teacher.py | 2 +- metagpt/roles/uml_meta_role_factory.py | 43 ++++++ metagpt/roles/uml_meta_role_options.py | 69 +++++++++ 11 files changed, 543 insertions(+), 128 deletions(-) create mode 100644 config/pattern/template.yaml create mode 100644 examples/fork_meta_role.py create mode 100644 metagpt/actions/meta_action.py delete mode 100644 metagpt/roles/meta_role.py create mode 100644 metagpt/roles/uml_meta_role_factory.py create mode 100644 metagpt/roles/uml_meta_role_options.py diff --git a/config/pattern/template.yaml b/config/pattern/template.yaml new file mode 100644 index 000000000..d148804f0 --- /dev/null +++ b/config/pattern/template.yaml @@ -0,0 +1,40 @@ +# Pattern Configuration Template +# Created By: mashenquan, 2023-8-7 +# File Name: template.yaml +# This template defines a set of structural standards for generating roles and action flows based on configurations. +# For more about UML 2.0 activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + +# project settings +startup: + requirement: "TeachingPlanRequirement" # Defines project initial requirement action + role: "Teacher" # Defines project role + investment: 3.0 # Defines the max project investment + n_round: 1 # Defines the max project round count + +# roles settings +roles: # A project can involve multiple roles. +- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. + name: "Lily" + profile: "{teaching_language} Teacher" + goal: "writing a {language} teaching plan part by part" + constraints: "writing in {language}" + role: "You are a {teaching_language} Teacher, named Lily, your goal is ..." + desc: "" + output_filename: "teaching_plan_demo.md" + requirement: ["TeachingPlanRequirement"] + templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. + - "Do ..." + - "Do ..." + # role's action settings + actions: # A role can have multiple actions. + - name: "" + topic: "Title" + language: "Chinese" + statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. + - "Statement: Find and return ..." + template_ix: 0 + rsp_begin_tag: "[..._BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. + rsp_end_tag: "[..._END]" # When asking, request the LLM to include the tag in the response. It's optional. + + + diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml index fbd8b4ae9..6a05bad96 100644 --- a/config/pattern/write_teaching_plan.yaml +++ b/config/pattern/write_teaching_plan.yaml @@ -1,51 +1,124 @@ -# `fork` role demo -- role_type: "fork" +# The `fork` role demo implements the flow of the code in `examples/write_teaching_plan.py`. + +# project settings +startup: + requirement: "TeachingPlanRequirement" # Defines project initial requirement action + role: "Teacher" + investment: 3.0 + n_round: 1 + +# roles settings +roles: # A project can involve multiple roles. +- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. name: "Lily" profile: "{teaching_language} Teacher" goal: "writing a {language} teaching plan part by part" constraints: "writing in {language}" + role: "You are a {teaching_language} Teacher, named Lily, your goal is writing a {teaching_language} teaching plan part by part, and the constraint is writing in {language}." desc: "" - actions: + output_filename: "teaching_plan_demo.md" + requirement: ["TeachingPlanRequirement"] + templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. + - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" + - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\nAnswer options: Using proper markdown format from second-level header format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" + actions: # 一个role可以有多个action - name: "" topic: "Title" language: "Chinese" - statements: - - "Statement: Find and return the title of the lesson only in markdown first-level header format, without anything else." + statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. + - "Statement: Find and return the title of the lesson only with \"# \" prefixed, without anything else." + template_ix: 0 + - name: "" + topic: "Teaching Hours" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. + rsp_end_tag: "[TEACHING_PLAN_END]" # When asking, request the LLM to include the tag in the response. It's optional. + - name: "" + topic: "Teaching Objectives" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Teaching Content" language: "Chinese" statements: - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar structures that appear in the textbook, as well as the listening materials and key points." - "Statement: \"Teaching Content\" must include more examples." - - name: "" - topic: "Teaching Time Allocation" - language: "Chinese" - statements: - - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Teaching Methods and Strategies" language: "Chinese" statements: - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, procedures, in detail." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Learning Activities" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Teaching Time Allocation" + language: "Chinese" + statements: + - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Assessment and Feedback" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Teaching Summary and Improvement" + language: "Chinese" + statements: [] + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Vocabulary Cloze" language: "Chinese" statements: - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers. The key-related vocabulary and phrases in the textbook content must all be included in the exercises." - - name: "" - topic: "Grammar Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Choice Questions" language: "Chinese" statements: - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create choice questions. 10 questions." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" + - name: "" + topic: "Grammar Questions" + language: "Chinese" + statements: + - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Translation Questions" language: "Chinese" statements: - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create translation questions. The translation should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers." + template_ix: 1 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" diff --git a/examples/fork_meta_role.py b/examples/fork_meta_role.py new file mode 100644 index 000000000..21e3b5f7c --- /dev/null +++ b/examples/fork_meta_role.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : fork_meta_role.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to possess the + ability to construct flows freely by concatenating symbols. Simultaneously, I am also striving to make + these symbols configurable and standardized, making the process of building flow structures more + convenient. This is a fork meta-role demo that implements the functionality of + `examples/write_teaching_plan.py`. +""" + +import asyncio +from pathlib import Path + +import aiofiles +import fire +import yaml + +from metagpt.actions.meta_action import MetaAction +from metagpt.logs import logger +from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory +from metagpt.roles.uml_meta_role_options import ProjectConfig +from metagpt.software_company import SoftwareCompany + + +async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): + """Run a startup. Be a teacher in education industry.""" + + demo_lesson = """ + UNIT 1 Making New Friends + TOPIC 1 Welcome to China! + Section A + + 1a Listen and number the following names. + Jane Mari Kangkang Michael + Look, listen and understand. Then practice the conversation. + Work in groups. Introduce yourself using + I ’m ... Then practice 1a + with your own hometown or the following places. + + 1b Listen and number the following names + Jane Michael Maria Kangkang + 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. + China the USA the UK Hong Kong Beijing + + 2a Look, listen and understand. Then practice the conversation + Hello! + Hello! + Hello! + Hello! Are you Maria? + No, I’m not. I’m Jane. + Oh, nice to meet you, Jane + Nice to meet you, too. + Hi, Maria! + Hi, Kangkang! + Welcome to China! + Thanks. + + 2b Work in groups. Make up a conversation with your own name and the + following structures. + A: Hello! / Good morning! / Hi! I’m ... Are you ... ? + B: ... + + 3a Listen, say and trace + Aa Bb Cc Dd Ee Ff Gg + + 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. + Aa Bb Cc Dd Ee Ff Gg + + 3c Match the big letters with the small ones. Then write them on the lines. + """ + + lesson = "" + if lesson_file is not None and Path(lesson_file).exists(): + async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: + lesson = await reader.read() + logger.info(f"Course content: {lesson}") + if not lesson: + logger.info("No course content provided, using the demo course.") + lesson = demo_lesson + + yaml_filename = kwargs["config"] + kwargs["lesson"] = lesson + + with open(yaml_filename, "r") as reader: + configs = yaml.safe_load(reader) + + startup_config = ProjectConfig(**configs) + roles = UMLMetaRoleFactory.create_roles(startup_config.roles, **kwargs) + company = SoftwareCompany() + company.hire(roles) + company.invest(startup_config.startup.investment) + company.start_project(lesson, role=startup_config.startup.role, + cause_by=MetaAction.get_action_type(startup_config.startup.requirement)) + await company.run(n_round=startup_config.startup.n_round) + + +def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): + """ + We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. + :param idea: lesson filename. + :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. + :param n_round: Reserved. + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + :return: + """ + asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) + + +if __name__ == '__main__': + """ + Formats: + ``` + python write_teaching_plan.py lesson_filename --teaching_language= --language= + ``` + If `lesson_filename` is not available, a demo lesson content will be used. + """ + fire.Fire(main) diff --git a/metagpt/actions/meta_action.py b/metagpt/actions/meta_action.py new file mode 100644 index 000000000..3f01b8c0f --- /dev/null +++ b/metagpt/actions/meta_action.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : meta_action.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + This file defines a meta action capable of generating arbitrary actions at runtime based on a + configuration file. +""" + +from typing import Type + +from metagpt.actions import Action +from metagpt.logs import logger +from metagpt.roles.uml_meta_role_options import MetaActionOptions +from metagpt.schema import Message + + +class MetaAction(Action): + def __init__(self, options: MetaActionOptions, llm=None, **kwargs): + super(MetaAction, self).__init__(options.name, kwargs.get("context"), llm=llm) + self.prompt = options.format_prompt(**kwargs) + self.options = options + self.kwargs = kwargs + + def __str__(self): + """Return `topic` value when str()""" + return self.options.topic + + def __repr__(self): + """Show `topic` value when debug""" + return self.options.topic + + async def run(self, messages, *args, **kwargs): + if len(messages) < 1 or not isinstance(messages[0], Message): + raise ValueError("Invalid args, a tuple of List[Message] is expected") + + logger.debug(self.prompt) + rsp = await self._aask(prompt=self.prompt) + logger.debug(rsp) + self._set_result(rsp) + return self.rsp + + def _set_result(self, rsp): + if self.options.rsp_begin_tag and self.options.rsp_begin_tag in rsp: + ix = rsp.index(self.options.rsp_begin_tag) + rsp = rsp[ix + len(self.options.rsp_begin_tag):] + if self.options.rsp_end_tag and self.options.rsp_end_tag in rsp: + ix = rsp.index(self.options.rsp_end_tag) + rsp = rsp[0:ix] + self.rsp = rsp.strip() + + @staticmethod + def get_action_type(topic: str): + """Create a runtime :class:`Action` subclass""" + action_type: Type["Action"] = type(topic, (Action,), {"name": topic}) + return action_type diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index a96aaf1be..625d98675 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,6 +4,8 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py +@Modified By: mashenquan, 2023-8-7. Modified get_by_actions() to support for dynamically generated Action classes + at runtime. """ from collections import defaultdict from typing import Iterable, Type @@ -80,8 +82,12 @@ class Memory: def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] + # Using the `type(obj).__name__` approach to support the runtime creation of requirement classes. + # See `MetaAction.get_action_type()` for more. + class_names = {type(k).__name__: k for k in self.index.keys()} for action in actions: - if action not in self.index: + if type(action).__name__ not in class_names: continue - rsp += self.index[action] + key = class_names[type(action).__name__] + rsp += self.index[key] return rsp diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index 1a69b9ca7..555bc8cf3 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -4,95 +4,124 @@ @Time : 2023/8/7 @Author : mashenquan @File : fork_meta_role.py -@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。这是一个`fork` meta-role demo，实现的是write_teaching_plan功能。 +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a + configuration file. """ +import re + +import aiofiles + +from metagpt.actions.meta_action import MetaAction +from metagpt.const import WORKSPACE_ROOT +from metagpt.logs import logger +from metagpt.roles import Role +from metagpt.roles.uml_meta_role_options import MetaActionOptions, UMLMetaRoleOptions +from metagpt.schema import Message -async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): - """Run a startup. Be a teacher in education industry.""" +class ForkMetaRole(Role): + """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" + def __init__(self, options, **kwargs): + """Initialize a `fork` style meta role - demo_lesson = """ - UNIT 1 Making New Friends - TOPIC 1 Welcome to China! - Section A + :param options: pattern yaml file data + :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + """ + opts = UMLMetaRoleOptions(**options) + global_variables = { + "name": Role.format_value(opts.name, kwargs), + "profile": Role.format_value(opts.profile, kwargs), + "goal": Role.format_value(opts.goal, kwargs), + "constraints": Role.format_value(opts.constraints, kwargs), + "desc": Role.format_value(opts.desc, kwargs), + "role": Role.format_value(opts.role, kwargs) + } + for k, v in kwargs.items(): + if k not in global_variables: + global_variables[k] = v - 1a Listen and number the following names. - Jane Mari Kangkang Michael - Look, listen and understand. Then practice the conversation. - Work in groups. Introduce yourself using - I ’m ... Then practice 1a - with your own hometown or the following places. + super(ForkMetaRole, self).__init__( + name=global_variables["name"], + profile=global_variables["profile"], + goal=global_variables["goal"], + constraints=global_variables["constraints"], + desc=global_variables["desc"], + **kwargs + ) + self.options = options + actions = [] + for m in opts.actions: + for k, v in m.items(): + v = Role.format_value(v, kwargs) + m[k] = v + for k, v in global_variables.items(): + if k not in m: + m[k] = v - 1b Listen and number the following names - Jane Michael Maria Kangkang - 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. - China the USA the UK Hong Kong Beijing + o = MetaActionOptions(**m) + o.set_default_template(opts.templates[o.template_ix]) - 2a Look, listen and understand. Then practice the conversation - Hello! - Hello! - Hello! - Hello! Are you Maria? - No, I’m not. I’m Jane. - Oh, nice to meet you, Jane - Nice to meet you, too. - Hi, Maria! - Hi, Kangkang! - Welcome to China! - Thanks. + act = MetaAction(options=o, llm=self._llm, **m) + actions.append(act) + self._init_actions(actions) + requirement_types = set() + for v in opts.requirement: + requirement_types.add(MetaAction.get_action_type(v)) + self._watch(requirement_types) - 2b Work in groups. Make up a conversation with your own name and the - following structures. - A: Hello! / Good morning! / Hi! I’m ... Are you ... ? - B: ... + async def _think(self) -> None: + """Everything will be done part by part.""" + if self._rc.todo is None: + self._set_state(0) + return - 3a Listen, say and trace - Aa Bb Cc Dd Ee Ff Gg + if self._rc.state + 1 < len(self._states): + self._set_state(self._rc.state + 1) + else: + self._rc.todo = None - 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. - Aa Bb Cc Dd Ee Ff Gg + async def _react(self) -> Message: + ret = Message(content="") + while True: + await self._think() + if self._rc.todo is None: + break + logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") + msg = await self._act() + if ret.content != '': + ret.content += "\n\n\n" + ret.content += msg.content + logger.info(ret.content) + await self.save(ret.content) + return ret - 3c Match the big letters with the small ones. Then write them on the lines. - """ + async def save(self, content): + """Save teaching plan""" + output_filename = self.options.get("output_filename") + if not output_filename: + return + filename = ForkMetaRole.new_file_name(output_filename) + pathname = WORKSPACE_ROOT / "teaching_plan" + pathname.mkdir(exist_ok=True) + pathname = pathname / filename + try: + async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + await writer.write(content) + except Exception as e: + logger.error(f'Save failed：{e}') + logger.info(f"Save to:{pathname}") - lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): - async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: - lesson = await reader.read() - logger.info(f"Course content: {lesson}") - if not lesson: - logger.info("No course content provided, using the demo course.") - lesson = demo_lesson - - - - company = SoftwareCompany() - company.hire([(*args, **kwargs)]) - company.invest(investment) - company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) - await company.run(n_round=1) - - -def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): - """ - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: lesson filename. - :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - :param n_round: Reserved. - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - :return: - """ - asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) - - -if __name__ == '__main__': - """ - Formats: - ``` - python write_teaching_plan.py lesson_filename --teaching_language= --language= - ``` - If `lesson_filename` is not available, a demo lesson content will be used. - """ - fire.Fire(main) + @staticmethod + def new_file_name(lesson_title, ext=".md"): + """Create a related file name based on `lesson_title` and `ext`.""" + # Define the special characters that need to be replaced. + illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' + # Replace the special characters with underscores. + filename = re.sub(illegal_chars, '_', lesson_title) + ext + return re.sub(r'_+', '_', filename) \ No newline at end of file diff --git a/metagpt/roles/meta_role.py b/metagpt/roles/meta_role.py deleted file mode 100644 index 1da180355..000000000 --- a/metagpt/roles/meta_role.py +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : meta_role.py -@Desc : 我试图将UML的一些符号概念引入到MetaGPT，使其具备通过符号拼接自由搭建flow的能力。同时我也尝试将这些符号做得配置化和标准化，让flow搭建流程更便捷。 - 分工参照UML 2.0 activity diagrams: `https://www.uml-diagrams.org/activity-diagrams.html` -""" -from typing import Dict, List - -from metagpt.roles import Role -from pydantic import BaseModel - -class UMLMetaRoleArgs(BaseModel): - role_type: str - name: str = "" - profile: str = "" - goal: str = "" - constraints: str = "" - desc: str = "" - actions: List - -class UMLMetaRole(Role): - """UML activity roles抽象父类""" - - def __init__(self, role_args: Dict): - """""" - self.role_args diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f79764324..1d65a7f26 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py -@Modified By: mashenquan, 2023-07-27, :class:`Role` + properties. +@Modified By: mashenquan, 2023-8-7, :class:`Role` + properties. """ from __future__ import annotations @@ -286,6 +286,8 @@ class Role: @staticmethod def format_value(value, options): """Fill parameters inside `value` with `options`.""" + if not isinstance(value, str): + return value if "{" not in value: return value @@ -295,7 +297,7 @@ class Role: except KeyError as e: logger.warning(f"Parameter is missing:{e}") for k, v in options.items(): - value = value.replace("{" + f"{k}" + "}", v) + value = value.replace("{" + f"{k}" + "}", str(v)) return value __DEFAULT_OPTIONS__ = { diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 95d54133b..24ede7402 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -5,7 +5,7 @@ @Author : mashenquan @File : teacher.py """ -from pathlib import Path + import aiofiles diff --git a/metagpt/roles/uml_meta_role_factory.py b/metagpt/roles/uml_meta_role_factory.py new file mode 100644 index 000000000..78f9689a2 --- /dev/null +++ b/metagpt/roles/uml_meta_role_factory.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : uml_meta_role_factory.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` +""" + +from metagpt.roles.fork_meta_role import ForkMetaRole +from metagpt.roles.uml_meta_role_options import UMLMetaRoleOptions + + +class UMLMetaRoleFactory: + """Factory of UML activity role classes""" + + @classmethod + def create_roles(cls, role_configs, **kwargs): + """Generate the flow of the project based on the configuration in the format of config/pattern/template.yaml. + + :param role_configs: `roles` field of template.yaml + :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` + + """ + roles = [] + for m in role_configs: + opt = UMLMetaRoleOptions(**m) + constructor = cls.CONSTRUCTORS.get(opt.role_type) + if constructor is None: + raise NotImplementedError( + f"{opt.role_type} is not implemented" + ) + r = constructor(m, **kwargs) + roles.append(r) + return roles + + CONSTRUCTORS = { + "fork": ForkMetaRole, + # TODO: add more activity node constructor here.. + } diff --git a/metagpt/roles/uml_meta_role_options.py b/metagpt/roles/uml_meta_role_options.py new file mode 100644 index 000000000..1d0fb322e --- /dev/null +++ b/metagpt/roles/uml_meta_role_options.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : uml_meta_role_options.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` +""" + +from typing import List, Dict + +from pydantic import BaseModel + + +# `startup` field of config/pattern/template.yaml +class StartupConfig(BaseModel): + requirement: str + role: str + investment: float = 3.0 + n_round: int = 3 + + +# config/pattern/template.yaml +class ProjectConfig(BaseModel): + startup: StartupConfig + roles: List[Dict] + + +# element of `actions` field of config/pattern/template.yaml +class MetaActionOptions(BaseModel): + topic: str + name: str = "" + language: str = "Chinese" + template_ix: int = 0 + statements: List[str] = [] + template: str = "" + rsp_begin_tag: str = "" + rsp_end_tag: str = "" + + def set_default_template(self, v): + if not self.template: + self.template = v + + def format_prompt(self, **kwargs): + statements = "\n".join(self.statements) + opts = kwargs.copy() + opts["statements"] = statements + + from metagpt.roles import Role + prompt = Role.format_value(self.template, opts) + return prompt + + +# element of `roles` field of config/pattern/template.yaml +class UMLMetaRoleOptions(BaseModel): + role_type: str + name: str = "" + profile: str = "" + goal: str = "" + role: str = "" + constraints: str = "" + desc: str = "" + templates: List[str] = [] + output_filename: str = "" + actions: List + requirement: List From 76ea924a051232ed6e2d3e4dcb84906f35ac09bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 15:25:26 +0800 Subject: [PATCH 026/378] feat: + unit test --- metagpt/roles/role.py | 5 ++ .../roles/test_uml_meta_role_factory.py | 61 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 tests/metagpt/roles/test_uml_meta_role_factory.py diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1d65a7f26..68baeccf5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -168,6 +168,11 @@ class Role: """Return role `constraints`, read only""" return self._setting.constraints + @property + def action_count(self): + """Return number of action""" + return len(self._actions) + def _get_prefix(self): """获取角色前缀""" if self._setting.desc: diff --git a/tests/metagpt/roles/test_uml_meta_role_factory.py b/tests/metagpt/roles/test_uml_meta_role_factory.py new file mode 100644 index 000000000..f59a30611 --- /dev/null +++ b/tests/metagpt/roles/test_uml_meta_role_factory.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_uml_meta_role_factory.py +""" +from typing import List, Dict + +from pydantic import BaseModel + +from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory + + +def test_create_roles(): + class Inputs(BaseModel): + roles: List + kwargs: Dict + + inputs = [ + { + "roles": [ + { + "role_type": "fork", + "name": "Lily", + "profile": "{teaching_language} Teacher", + "goal": "writing a {language} teaching plan part by part", + "constraints": "writing in {language}", + "role": "You are a {teaching_language} Teacher, named Lily.", + "desc": "", + "output_filename": "teaching_plan_demo.md", + "requirement": ["TeachingPlanRequirement"], + "templates": ["Do 1 {statements}", "Do 2 {statements}"], + "actions": [ + { + "name": "", + "topic": "Title", + "language": "Chinese", + "statements": ["statement 1", "statement 2"]} + ], + "template_ix": 0 + } + ], + "kwargs": { + "teaching_language": "AA", + "language": "BB", + } + } + ] + + for i in inputs: + seed = Inputs(**i) + roles = UMLMetaRoleFactory.create_roles(seed.roles, **seed.kwargs) + assert len(roles) == 1 + assert "{" not in roles[0].profile + assert "{" not in roles[0].goal + assert roles[0].action_count == 1 + + +if __name__ == '__main__': + test_create_roles() From 1526680fc4b867668a84bf3813d02aefa0004044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 18:02:49 +0800 Subject: [PATCH 027/378] feat: + unit test --- .../roles/test_uml_meta_role_options.py | 40 +++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/metagpt/roles/test_uml_meta_role_options.py diff --git a/tests/metagpt/roles/test_uml_meta_role_options.py b/tests/metagpt/roles/test_uml_meta_role_options.py new file mode 100644 index 000000000..1eb66c50e --- /dev/null +++ b/tests/metagpt/roles/test_uml_meta_role_options.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_uml_meta_role_options.py +""" +from typing import List + +from pydantic import BaseModel + +from metagpt.roles.uml_meta_role_options import MetaActionOptions + + +def test_set_default_template(): + class Inputs(BaseModel): + statements: List + template: str + expect_prompt: str + + inputs = [ + { + "statements": ["Statement: 1", "Statement: 2"], + "template": "{statements}", + "expect_prompt": "Statement: 1\nStatement: 2" + } + ] + + for i in inputs: + seed = Inputs(**i) + opt = MetaActionOptions(topic="", statements=seed.statements) + assert opt.template == "" + opt.set_default_template(seed.template) + assert opt.template == seed.template + kwargs = {} + assert opt.format_prompt(**kwargs) == seed.expect_prompt + + +if __name__ == '__main__': + test_set_default_template() From 25f461b6a4ae2137c463202ce39769a107bbab89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 19:02:57 +0800 Subject: [PATCH 028/378] feat: + unit test --- tests/metagpt/roles/test_fork_meta_role.py | 90 ++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 tests/metagpt/roles/test_fork_meta_role.py diff --git a/tests/metagpt/roles/test_fork_meta_role.py b/tests/metagpt/roles/test_fork_meta_role.py new file mode 100644 index 000000000..b2659330d --- /dev/null +++ b/tests/metagpt/roles/test_fork_meta_role.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_fork_meta_role.py +""" +from typing import Dict + +from pydantic import BaseModel + +from metagpt.roles.fork_meta_role import ForkMetaRole + + +def test_creat_role(): + class Inputs(BaseModel): + role: Dict + action_count: int + + inputs = [ + { + "role": { + "role_type": "fork", + "name": "Lily", + "profile": "{teaching_language} Teacher", + "goal": "writing a {language} teaching plan part by part", + "constraints": "writing in {language}", + "role": "You are a {teaching_language} Teacher, named Lily, your goal is writing a {" + "teaching_language} teaching plan part by part, and the constraint is writing in {language}.", + "desc": "", + "output_filename": "teaching_plan_demo.md", + "requirement": ["TeachingPlanRequirement"], + "templates": [ + "Do not refer to the context of the previous conversation records, start the conversation " + "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" + "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " + "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " + "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " + "responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: " + "Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[" + "LESSON_BEGIN]\n{lesson}\n[LESSON_END]", + "Do not refer to the context of the previous conversation records, start the conversation " + "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" + "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " + "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " + "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " + "responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part " + "of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: " + "Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" " + "tags.\nAnswer options: Using proper markdown format from second-level header " + "format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END] " + ], + "actions": [ + { + "name": "", + "topic": "Title", + "language": "Chinese", + "statements": [ + "Statement: Find and return the title of the lesson only with \"# \" prefixed, without " + "anything else."], + "template_ix": 0}, + { + "name": "", + "topic": "Teaching Hours", + "language": "Chinese", + "statements": [], + "template_ix": 1, + "rsp_begin_tag": "[TEACHING_PLAN_BEGIN]", + "rsp_end_tag": "[TEACHING_PLAN_END]"} + ] + }, + "action_count": 2 + } + ] + + for i in inputs: + seed = Inputs(**i) + kwargs = { + "teaching_language": "AA", + "language": "BB" + } + role = ForkMetaRole(seed.role, **kwargs) + assert role.action_count == 2 + assert "{" not in role.profile + assert "{" not in role.goal + assert "{" not in role.constraints + + +if __name__ == '__main__': + test_creat_role() From b0209f3d419e2feffa099902ba4f76429963e6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 8 Aug 2023 20:10:40 +0800 Subject: [PATCH 029/378] feat: + unit test --- tests/metagpt/actions/test_meta_action.py | 51 +++++++++++++++++++++++ 1 file changed, 51 insertions(+) create mode 100644 tests/metagpt/actions/test_meta_action.py diff --git a/tests/metagpt/actions/test_meta_action.py b/tests/metagpt/actions/test_meta_action.py new file mode 100644 index 000000000..cbaf3456c --- /dev/null +++ b/tests/metagpt/actions/test_meta_action.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/8 +@Author : mashenquan +@File : test_meta_action.py +""" +from typing import Dict + +from pydantic import BaseModel + +from metagpt.actions.meta_action import MetaAction +from metagpt.roles.uml_meta_role_options import MetaActionOptions + + +def test_meta_action_create(): + class Inputs(BaseModel): + options: Dict + kwargs: Dict + expect_class_name: str + expect_prompt: str + + inputs = [ + { + "options": { + "topic": "TOPIC_A", + "name": "A", + "language": "XX", + "template_ix": 0, + "statements": ["Statement A", "Statement B"], + "template": "{statements}", + "rsp_begin_tag": "", + "rsp_end_tag": "" + }, + "kwargs": {}, + "expect_class_name": "TOPIC_A", + "expect_prompt": "\n".join(["Statement A", "Statement B"]), + } + ] + + for i in inputs: + seed = Inputs(**i) + opt = MetaActionOptions(**seed.options) + act = MetaAction(opt, **seed.kwargs) + assert seed.expect_prompt == act.prompt + t = MetaAction.get_action_type(seed.expect_class_name) + assert t.__name__ == seed.expect_class_name + + +if __name__ == '__main__': + test_meta_action_create() From f8bc4e462575622927a2b7b022468bd95a38a8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 10:48:40 +0800 Subject: [PATCH 030/378] feat: test pass --- config/pattern/write_teaching_plan.yaml | 2 +- ...{fork_meta_role.py => fork_meta_role_write_teaching_plan.py} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename examples/{fork_meta_role.py => fork_meta_role_write_teaching_plan.py} (100%) diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml index 6a05bad96..357717908 100644 --- a/config/pattern/write_teaching_plan.yaml +++ b/config/pattern/write_teaching_plan.yaml @@ -16,7 +16,7 @@ roles: # A project can involve multiple r constraints: "writing in {language}" role: "You are a {teaching_language} Teacher, named Lily, your goal is writing a {teaching_language} teaching plan part by part, and the constraint is writing in {language}." desc: "" - output_filename: "teaching_plan_demo.md" + output_filename: "teaching_plan_demo" requirement: ["TeachingPlanRequirement"] templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" diff --git a/examples/fork_meta_role.py b/examples/fork_meta_role_write_teaching_plan.py similarity index 100% rename from examples/fork_meta_role.py rename to examples/fork_meta_role_write_teaching_plan.py From 9e91142c4ef30ef08ff9552a1b57bd954540e6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 13:46:22 +0800 Subject: [PATCH 031/378] fixbug: Align parameters with the parent class --- metagpt/actions/design_api.py | 2 +- metagpt/actions/project_management.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..48fa7171a 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -135,7 +135,7 @@ class WriteDesign(Action): self._save_prd(docs_path, resources_path, context[-1].content) self._save_system_design(docs_path, resources_path, content) - async def run(self, context): + async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..80b891bb8 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -115,7 +115,7 @@ class WriteTasks(Action): requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) - async def run(self, context): + async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) self._save(context, rsp) From 9e9e0d56e56ba2206f090bc040c14b580e1254d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 13:55:07 +0800 Subject: [PATCH 032/378] fixbug: Align parameters with the parent class --- metagpt/actions/design_api.py | 1 + metagpt/actions/project_management.py | 1 + 2 files changed, 2 insertions(+) diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 48fa7171a..cf23e6ad1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ import shutil from pathlib import Path diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 80b891bb8..16473ff01 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ from typing import List, Tuple From 35470dcee4d4635b798f03ccd6154467d65cb57e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 14:40:43 +0800 Subject: [PATCH 033/378] fixbug: empty string causes aiofiles.open exepctition --- examples/write_teaching_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index da97a5463..86125f090 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -62,7 +62,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * """ lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): + if lesson_file and Path(lesson_file).exists(): async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: lesson = await reader.read() logger.info(f"Course content: {lesson}") From ed56aab79cc90ede26ca4993476e260721093fce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 14:54:08 +0800 Subject: [PATCH 034/378] fixbug: empty string causes aiofiles.open exepctition --- config/pattern/write_teaching_plan.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml index 357717908..5b5f2af77 100644 --- a/config/pattern/write_teaching_plan.yaml +++ b/config/pattern/write_teaching_plan.yaml @@ -26,8 +26,10 @@ roles: # A project can involve multiple r topic: "Title" language: "Chinese" statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. - - "Statement: Find and return the title of the lesson only with \"# \" prefixed, without anything else." + - "Statement: Find and return the title of the lesson only with \"# \" string prefixed, without anything else." template_ix: 0 + rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" + rsp_end_tag: "[TEACHING_PLAN_END]" - name: "" topic: "Teaching Hours" language: "Chinese" From b786bce4b9daf573f8f5eb4adc78f9f453ea3e4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 14:54:30 +0800 Subject: [PATCH 035/378] fixbug: empty string causes aiofiles.open exepctition --- examples/fork_meta_role_write_teaching_plan.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/fork_meta_role_write_teaching_plan.py b/examples/fork_meta_role_write_teaching_plan.py index 21e3b5f7c..d1a6e0070 100644 --- a/examples/fork_meta_role_write_teaching_plan.py +++ b/examples/fork_meta_role_write_teaching_plan.py @@ -73,7 +73,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * """ lesson = "" - if lesson_file is not None and Path(lesson_file).exists(): + if lesson_file and Path(lesson_file).exists(): async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: lesson = await reader.read() logger.info(f"Course content: {lesson}") From 13a91349f78322a7f3df87e36b266ed501398c3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 16:37:29 +0800 Subject: [PATCH 036/378] fixbug: cannot find metagpt module --- examples/llm_hello_world.py | 5 ++++- examples/search_google.py | 5 ++++- examples/search_kb.py | 5 ++++- examples/search_with_specific_engine.py | 9 ++++++++- examples/write_teaching_plan.py | 6 +++++- 5 files changed, 25 insertions(+), 5 deletions(-) diff --git a/examples/llm_hello_world.py b/examples/llm_hello_world.py index 3ba03eea0..329247afc 100644 --- a/examples/llm_hello_world.py +++ b/examples/llm_hello_world.py @@ -4,9 +4,12 @@ @Time : 2023/5/6 14:13 @Author : alexanderwu @File : llm_hello_world.py +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.llm import LLM, Claude from metagpt.logs import logger diff --git a/examples/search_google.py b/examples/search_google.py index 9e9521b9c..df45c29ea 100644 --- a/examples/search_google.py +++ b/examples/search_google.py @@ -4,10 +4,13 @@ @Time : 2023/5/7 18:32 @Author : alexanderwu @File : search_google.py +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.roles import Searcher diff --git a/examples/search_kb.py b/examples/search_kb.py index b6f7d87a0..449099380 100644 --- a/examples/search_kb.py +++ b/examples/search_kb.py @@ -2,9 +2,12 @@ # -*- coding: utf-8 -*- """ @File : search_kb.py +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. """ import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.const import DATA_PATH from metagpt.document_store import FaissStore from metagpt.logs import logger diff --git a/examples/search_with_specific_engine.py b/examples/search_with_specific_engine.py index 7cc431cd4..4423011e4 100644 --- a/examples/search_with_specific_engine.py +++ b/examples/search_with_specific_engine.py @@ -1,5 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023-8-9, fix-bug: cannot find metagpt module. +""" import asyncio - +from pathlib import Path +import sys +sys.path.append(str(Path(__file__).resolve().parent.parent)) from metagpt.roles import Searcher from metagpt.tools import SearchEngineType diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 86125f090..5ab7d3ab5 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -1,11 +1,15 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Modified By: mashenquan, 2023-07-27, + `industry` concept +@Modified By: mashenquan, 2023-07-27, + write teaching plan flow demo + """ import asyncio from pathlib import Path +import sys + +sys.path.append(str(Path(__file__).resolve().parent.parent)) import aiofiles import fire from metagpt.logs import logger From 63678de181096b73f6680b36bcc97e38c4dca113 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 9 Aug 2023 20:48:25 +0800 Subject: [PATCH 037/378] feat: add more text formatting options --- metagpt/actions/azure_tts.py | 42 +++++++++++++++++++------ tests/metagpt/actions/test_azure_tts.py | 30 +++++++++++++++--- 2 files changed, 58 insertions(+), 14 deletions(-) diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py index f528ba001..3520de8b4 100644 --- a/metagpt/actions/azure_tts.py +++ b/metagpt/actions/azure_tts.py @@ -4,11 +4,13 @@ @Time : 2023/6/9 22:22 @Author : Leo Xiao @File : azure_tts.py +@Modified By: mashenquan, 2023-8-9, add more text formatting options """ from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer from metagpt.actions.action import Action from metagpt.config import Config +from metagpt.const import WORKSPACE_ROOT class AzureTTS(Action): @@ -17,7 +19,7 @@ class AzureTTS(Action): self.config = Config() # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, role, text, output_file): + def synthesize_speech(self, lang, voice, text, output_file): subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') region = self.config.get('AZURE_TTS_REGION') speech_config = SpeechConfig( @@ -29,25 +31,47 @@ class AzureTTS(Action): speech_config=speech_config, audio_config=audio_config) - # if voice=="zh-CN-YunxiNeural": + # More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice ssml_string = f""" - - {text} - + {text} """ - synthesizer.speak_ssml_async(ssml_string).get() + return synthesizer.speak_ssml_async(ssml_string).get() + @staticmethod + def role_style_text(role, style, text): + return f'{text}' + + @staticmethod + def role_text(role, text): + return f'{text}' + + @staticmethod + def style_text(style, text): + return f'{text}' if __name__ == "__main__": azure_tts = AzureTTS("azure_tts") + text = """ + 女儿看见父亲走了进来，问道： + + “您来的挺快的，怎么过来的？” + + 父亲放下手提包，说： + + “刚打车过来的，路上还挺顺畅。” + + """ + path = WORKSPACE_ROOT / "tts" + path.mkdir(exist_ok=True, parents=True) + filename = path / "output.wav" azure_tts.synthesize_speech( "zh-CN", "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") + text=AzureTTS.role_style_text(role="Boy", style="affectionate", text="你好，我是卡卡"), + output_file=str(filename) + ) diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/actions/test_azure_tts.py index b5a333af2..2145f7133 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/actions/test_azure_tts.py @@ -4,18 +4,38 @@ @Time : 2023/7/1 22:50 @Author : alexanderwu @File : test_azure_tts.py +@Modified By: mashenquan, 2023-8-9, add more text formatting options """ from metagpt.actions.azure_tts import AzureTTS +from metagpt.const import WORKSPACE_ROOT def test_azure_tts(): azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( + text = """ + 女儿看见父亲走了进来，问道： + + “您来的挺快的，怎么过来的？” + + 父亲放下手提包，说： + + “Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.” + + """ + path = WORKSPACE_ROOT / "tts" + path.mkdir(exist_ok=True, parents=True) + filename = path / "girl.wav" + result = azure_tts.synthesize_speech( "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") + "zh-CN-XiaomoNeural", + text=text, + output_file=str(filename)) + + print(result) # 运行需要先配置 SUBSCRIPTION_KEY # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 + + +if __name__ == '__main__': + test_azure_tts() From de610df25d310211c2f235c9a7a79fc4162c219e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 17 Aug 2023 20:41:07 +0800 Subject: [PATCH 038/378] feat: +OAS framework --- metagpt/actions/azure_tts.py | 53 -------- metagpt/tools/azure_tts.py | 114 ++++++++++++++++++ metagpt/tools/hello.py | 27 +++++ metagpt/tools/metagpt_openapi_svc.py | 20 +++ metagpt/utils/common.py | 13 ++ requirements.txt | 4 +- spec/metagpt_openapi.yaml | 64 ++++++++++ spec/openapi.yaml | 35 ++++++ .../{actions => tools}/test_azure_tts.py | 7 +- 9 files changed, 282 insertions(+), 55 deletions(-) delete mode 100644 metagpt/actions/azure_tts.py create mode 100644 metagpt/tools/azure_tts.py create mode 100644 metagpt/tools/hello.py create mode 100644 metagpt/tools/metagpt_openapi_svc.py create mode 100644 spec/metagpt_openapi.yaml create mode 100644 spec/openapi.yaml rename tests/metagpt/{actions => tools}/test_azure_tts.py (67%) diff --git a/metagpt/actions/azure_tts.py b/metagpt/actions/azure_tts.py deleted file mode 100644 index f528ba001..000000000 --- a/metagpt/actions/azure_tts.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/6/9 22:22 -@Author : Leo Xiao -@File : azure_tts.py -""" -from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer - -from metagpt.actions.action import Action -from metagpt.config import Config - - -class AzureTTS(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) - self.config = Config() - - # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, role, text, output_file): - subscription_key = self.config.get('AZURE_TTS_SUBSCRIPTION_KEY') - region = self.config.get('AZURE_TTS_REGION') - speech_config = SpeechConfig( - subscription=subscription_key, region=region) - - speech_config.speech_synthesis_voice_name = voice - audio_config = AudioConfig(filename=output_file) - synthesizer = SpeechSynthesizer( - speech_config=speech_config, - audio_config=audio_config) - - # if voice=="zh-CN-YunxiNeural": - ssml_string = f""" - - - - {text} - - - - """ - - synthesizer.speak_ssml_async(ssml_string).get() - - -if __name__ == "__main__": - azure_tts = AzureTTS("azure_tts") - azure_tts.synthesize_speech( - "zh-CN", - "zh-CN-YunxiNeural", - "Boy", - "你好，我是卡卡", - "output.wav") diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py new file mode 100644 index 000000000..19d7c2ab1 --- /dev/null +++ b/metagpt/tools/azure_tts.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : azure_tts.py +@Desc : azure TTS openapi, which provides text-to-speech functionality +""" +from pathlib import Path +from uuid import uuid4 +import base64 +import sys + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initalize_enviroment +from metagpt.logs import logger + +from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer +import os + + +class AzureTTS: + """Azure Text-to-Speech""" + + def __init__(self, subscription_key, region): + """ + :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` + :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. + """ + self.subscription_key = subscription_key if subscription_key else os.environ.get('AZURE_TTS_SUBSCRIPTION_KEY') + self.region = region if region else os.environ.get('AZURE_TTS_REGION') + + # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles + def synthesize_speech(self, lang, voice, text, output_file): + speech_config = SpeechConfig( + subscription=self.subscription_key, region=self.region) + speech_config.speech_synthesis_voice_name = voice + audio_config = AudioConfig(filename=output_file) + synthesizer = SpeechSynthesizer( + speech_config=speech_config, + audio_config=audio_config) + + # More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice + ssml_string = "" \ + f"{text}" + + return synthesizer.speak_ssml_async(ssml_string).get() + + @staticmethod + def role_style_text(role, style, text): + return f'{text}' + + @staticmethod + def role_text(role, text): + return f'{text}' + + @staticmethod + def style_text(style, text): + return f'{text}' + + +# Export +def openapi_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): + """openapi/tts/azsure + For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + + :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery` + :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param text: Text to convert + :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` + :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. + :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. + + """ + if not text: + return "" + + if not lang: + lang = "zh-CN" + if not voice: + voice = "zh-CN-XiaomoNeural" + if not role: + role = "Girl" + if not style: + style = "affectionate" + if not subscription_key: + subscription_key = os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") + if not region: + region = os.environ.get("AZURE_TTS_REGION") + + xml_value = AzureTTS.role_style_text(role=role, style=style, text=text) + tts = AzureTTS(subscription_key=subscription_key, region=region) + filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav") + try: + tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) + with open(str(filename), mode="rb") as reader: + data = reader.read() + base64_string = base64.b64encode(data).decode('utf-8') + filename.unlink() + except Exception as e: + logger.error(f"text:{text}, error:{e}") + return "" + + return base64_string + + +if __name__ == "__main__": + initalize_enviroment() + + v = openapi_azsure_tts("测试，test") + print(v) diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py new file mode 100644 index 000000000..686fba34b --- /dev/null +++ b/metagpt/tools/hello.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/2 16:03 +@Author : mashenquan +@File : hello.py +@Desc : Implement the OpenAPI Specification 3.0 demo and use the following command to test the HTTP service: + + curl -X 'POST' \ + 'http://localhost:8080/openapi/greeting/dave' \ + -H 'accept: text/plain' \ + -H 'Content-Type: application/json' \ + -d '{}' +""" + +import connexion + + +# openapi implement +def post_greeting(name: str) -> str: + return f"Hello {name}\n" + + +if __name__ == "__main__": + app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app.add_api("openapi.yaml", arguments={"title": "Hello World Example"}) + app.run(port=8080) diff --git a/metagpt/tools/metagpt_openapi_svc.py b/metagpt/tools/metagpt_openapi_svc.py new file mode 100644 index 000000000..94d935625 --- /dev/null +++ b/metagpt/tools/metagpt_openapi_svc.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : metagpt_openapi_svc.py +@Desc : MetaGPT OpenAPI REST API service +""" +from pathlib import Path +import sys +import connexion +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initalize_enviroment + +if __name__ == "__main__": + initalize_enviroment() + + app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app.add_api("metagpt_openapi.yaml") + app.run(port=8080) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index 7f090cf63..b15c1d186 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -4,14 +4,18 @@ @Time : 2023/4/29 16:07 @Author : alexanderwu @File : common.py +@Modified By: mashenquan, 2023-8-17, add `initalize_enviroment()` to load `config/config.yaml` to `os.environ` """ import ast import contextlib import inspect import os import re +from pathlib import Path from typing import List, Tuple +import yaml + from metagpt.logs import logger @@ -254,3 +258,12 @@ def parse_recipient(text): pattern = r"## Send To:\s*([A-Za-z]+)\s*?" # hard code for now recipient = re.search(pattern, text) return recipient.group(1) if recipient else "" + + +def initalize_enviroment(): + """Load `config/config.yaml` to `os.environ`""" + yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" + with open(str(yaml_file_path), "r") as yaml_file: + data = yaml.safe_load(yaml_file) + for k, v in data.items(): + os.environ[k] = str(v) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index c18145b98..eef7464ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,4 +36,6 @@ anthropic==0.3.6 typing-inspect==0.8.0 typing_extensions==4.5.0 libcst==1.0.1 -qdrant-client==1.4.0 \ No newline at end of file +qdrant-client==1.4.0 +connexion[swagger-ui] +aiohttp_jinja2 \ No newline at end of file diff --git a/spec/metagpt_openapi.yaml b/spec/metagpt_openapi.yaml new file mode 100644 index 000000000..0bb6ae7bf --- /dev/null +++ b/spec/metagpt_openapi.yaml @@ -0,0 +1,64 @@ +openapi: "3.0.0" + +info: + title: "MetaGPT Export OpenAPIs" + version: "1.0" +servers: + - url: "/openapi" + +paths: + /tts/azsure: + post: + summary: "Convert Text to Base64-encoded .wav File Stream" + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + operationId: azure_tts.openapi_azsure_tts + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - text + properties: + text: + type: string + description: Text to convert + lang: + type: string + description: The language code or locale, e.g., en-US (English - United States) + default: "zh-CN" + voice: + type: string + description: "Voice style, see: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts), [Voice Gallery](https://speech.microsoft.com/portal/voicegallery)" + default: "zh-CN-XiaomoNeural" + style: + type: string + description: "Speaking style to express different emotions. For more details, checkout: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + default: "affectionate" + role: + type: string + description: "Role to specify age and gender. For more details, checkout: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + default: "Girl" + subscription_key: + type: string + description: "Key used to access Azure AI service API, see: [Azure Portal](https://portal.azure.com/) > `Resource Management` > `Keys and Endpoint`" + default: "" + region: + type: string + description: "Location (or region) of your resource, see: [Azure Portal](https://portal.azure.com/) > `Resource Management` > `Keys and Endpoint`" + default: "" + responses: + '200': + description: "Base64-encoded .wav file data if successful, otherwise an empty string." + content: + application/json: + schema: + type: object + properties: + result: + type: string + '400': + description: Bad Request + '500': + description: Bad Request \ No newline at end of file diff --git a/spec/openapi.yaml b/spec/openapi.yaml new file mode 100644 index 000000000..bc291b7db --- /dev/null +++ b/spec/openapi.yaml @@ -0,0 +1,35 @@ +openapi: "3.0.0" + +info: + title: Hello World + version: "1.0" +servers: + - url: /openapi + +paths: + /greeting/{name}: + post: + summary: Generate greeting + description: Generates a greeting message. + operationId: hello.post_greeting + responses: + 200: + description: greeting response + content: + text/plain: + schema: + type: string + example: "hello dave!" + parameters: + - name: name + in: path + description: Name of the person to greet. + required: true + schema: + type: string + example: "dave" + requestBody: + content: + application/json: + schema: + type: object \ No newline at end of file diff --git a/tests/metagpt/actions/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py similarity index 67% rename from tests/metagpt/actions/test_azure_tts.py rename to tests/metagpt/tools/test_azure_tts.py index b5a333af2..667e32d01 100644 --- a/tests/metagpt/actions/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -4,8 +4,13 @@ @Time : 2023/7/1 22:50 @Author : alexanderwu @File : test_azure_tts.py +@Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ -from metagpt.actions.azure_tts import AzureTTS +import sys +from pathlib import Path + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.tools.azure_tts import AzureTTS def test_azure_tts(): From eb232efdfc438c0a4425fca9f6ad48f23f9825ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 17 Aug 2023 20:55:53 +0800 Subject: [PATCH 039/378] feat: rename --- metagpt/tools/azure_tts.py | 4 ++-- .../{metagpt_openapi_svc.py => metagpt_oas3_api_svc.py} | 6 +++--- spec/{metagpt_openapi.yaml => metagpt_oas3_api.yaml} | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) rename metagpt/tools/{metagpt_openapi_svc.py => metagpt_oas3_api_svc.py} (77%) rename spec/{metagpt_openapi.yaml => metagpt_oas3_api.yaml} (96%) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 19d7c2ab1..035a85108 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -61,8 +61,8 @@ class AzureTTS: # Export -def openapi_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): - """openapi/tts/azsure +def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): + """oas3/tts/azsure For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` diff --git a/metagpt/tools/metagpt_openapi_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py similarity index 77% rename from metagpt/tools/metagpt_openapi_svc.py rename to metagpt/tools/metagpt_oas3_api_svc.py index 94d935625..921629d8c 100644 --- a/metagpt/tools/metagpt_openapi_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -3,8 +3,8 @@ """ @Time : 2023/8/17 @Author : mashenquan -@File : metagpt_openapi_svc.py -@Desc : MetaGPT OpenAPI REST API service +@File : metagpt_oas3_api_svc.py +@Desc : MetaGPT OpenAPI Specification 3.0 REST API service """ from pathlib import Path import sys @@ -16,5 +16,5 @@ if __name__ == "__main__": initalize_enviroment() app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') - app.add_api("metagpt_openapi.yaml") + app.add_api("metagpt_oas3_api.yaml") app.run(port=8080) diff --git a/spec/metagpt_openapi.yaml b/spec/metagpt_oas3_api.yaml similarity index 96% rename from spec/metagpt_openapi.yaml rename to spec/metagpt_oas3_api.yaml index 0bb6ae7bf..5a3e6923b 100644 --- a/spec/metagpt_openapi.yaml +++ b/spec/metagpt_oas3_api.yaml @@ -4,14 +4,14 @@ info: title: "MetaGPT Export OpenAPIs" version: "1.0" servers: - - url: "/openapi" + - url: "/oas3" paths: /tts/azsure: post: summary: "Convert Text to Base64-encoded .wav File Stream" description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - operationId: azure_tts.openapi_azsure_tts + operationId: azure_tts.oas3_azsure_tts requestBody: required: true content: From 60245fbe902287cc40ea0643d7764da0f50da29a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 17 Aug 2023 21:51:50 +0800 Subject: [PATCH 040/378] feat: +openai text-to-image --- metagpt/tools/azure_tts.py | 6 +- metagpt/tools/openai_text_2_image.py | 100 +++++++++++++++++++++++++++ spec/metagpt_oas3_api.yaml | 42 ++++++++++- 3 files changed, 143 insertions(+), 5 deletions(-) create mode 100644 metagpt/tools/openai_text_2_image.py diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 035a85108..5d0001b27 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -4,7 +4,7 @@ @Time : 2023/8/17 @Author : mashenquan @File : azure_tts.py -@Desc : azure TTS openapi, which provides text-to-speech functionality +@Desc : azure TTS OAS3 api, which provides text-to-speech functionality """ from pathlib import Path from uuid import uuid4 @@ -69,7 +69,7 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery` :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` - :param text: Text to convert + :param text: The text used for voice conversion. :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. @@ -110,5 +110,5 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key if __name__ == "__main__": initalize_enviroment() - v = openapi_azsure_tts("测试，test") + v = oas3_azsure_tts("测试，test") print(v) diff --git a/metagpt/tools/openai_text_2_image.py b/metagpt/tools/openai_text_2_image.py new file mode 100644 index 000000000..3d2a2bbfc --- /dev/null +++ b/metagpt/tools/openai_text_2_image.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : openai_text_2_image.py +@Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality. +""" +import base64 +import os +import sys +from pathlib import Path +from typing import List + +import requests +from pydantic import BaseModel + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initalize_enviroment +from metagpt.logs import logger + + +class OpenAIText2Image: + def __init__(self, openai_api_key): + """ + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + """ + self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + + def text_2_image(self, text, size_type="1024x1024"): + """Text to image + + :param text: The text used for image conversion. + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + + class ImageUrl(BaseModel): + url: str + + class ImageResult(BaseModel): + data: List[ImageUrl] + created: int + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.openai_api_key}" + } + data = {"prompt": text, "n": 1, "size": size_type} + try: + response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + result = ImageResult(**response.json()) + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + if len(result.data) > 0: + return OpenAIText2Image.get_image_data(result.data[0].url) + return "" + + @staticmethod + def get_image_data(url): + """Fetch image data from a URL and encode it as Base64 + + :param url: Image url + :return: Base64-encoded image data. + """ + try: + response = requests.get(url) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + image_data = response.content + base64_image = base64.b64encode(image_data).decode("utf-8") + return base64_image + + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + + +# Export +def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key=""): + """Text to image + + :param text: The text used for image conversion. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + if not text: + return "" + if not openai_api_key: + openai_api_key = os.environ.get("OPENAI_API_KEY") + return OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) + + +if __name__ == "__main__": + initalize_enviroment() + + v = oas3_openai_text_2_image("Panda emoji") + print(v) diff --git a/spec/metagpt_oas3_api.yaml b/spec/metagpt_oas3_api.yaml index 5a3e6923b..70c15d590 100644 --- a/spec/metagpt_oas3_api.yaml +++ b/spec/metagpt_oas3_api.yaml @@ -59,6 +59,44 @@ paths: result: type: string '400': - description: Bad Request + description: "Bad Request" '500': - description: Bad Request \ No newline at end of file + description: "Internal Server Error" + + /txt2img/openai: + post: + summary: "Convert Text to Base64-encoded Image Data Stream" + operationId: openai_text_2_image.oas3_openai_text_2_image + requestBody: + required: true + content: + application/json: + schema: + type: object + properties: + text: + type: string + description: "The text used for image conversion." + size_type: + type: string + enum: ["256x256", "512x512", "1024x1024"] + default: "1024x1024" + description: "Size of the generated image." + openai_api_key: + type: string + default: "" + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + responses: + '200': + description: "Base64-encoded image data." + content: + application/json: + schema: + type: object + properties: + image_data: + type: string + '400': + description: "Bad Request" + '500': + description: "Internal Server Error" \ No newline at end of file From 2513cca46b4c51d0f4adb9a1a5dce637c0087a51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 10:57:52 +0800 Subject: [PATCH 041/378] feat: +ai-plugin --- .well-known/MetaGPT-logo.png | Bin 0 -> 50622 bytes .well-known/ai-plugin.json | 18 ++++++++++++++++++ {spec => .well-known}/metagpt_oas3_api.yaml | 4 +++- {spec => .well-known}/openapi.yaml | 0 metagpt/tools/azure_tts.py | 2 +- metagpt/tools/hello.py | 2 +- metagpt/tools/metagpt_oas3_api_svc.py | 6 +++--- metagpt/tools/openai_text_2_image.py | 4 ++-- metagpt/utils/common.py | 4 ++-- 9 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 .well-known/MetaGPT-logo.png create mode 100644 .well-known/ai-plugin.json rename {spec => .well-known}/metagpt_oas3_api.yaml (96%) rename {spec => .well-known}/openapi.yaml (100%) diff --git a/.well-known/MetaGPT-logo.png b/.well-known/MetaGPT-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..159517fcd4f62049f43eec4db62e1770d189ae89 GIT binary patch literal 50622 zcmeEt^;eVs`~Of0QNkdUvJh!SNeKl(r9@%W2x(!A(MSyy0Ra&N0g)0VH^#_~+@!l> zz!=?QG$Y2~v)A{}_@4I<_qp%gXXk9s>v_fFdR&iZq=Ei(HrDH`004mP<%?%;0D!aq z{<~P1=x>fjI8*7j^X@OqyZ`_Ro&PR|9F?R3`b!3{H_z1pMLqm0^bhBpp6WdX07_yn z9NRDg0L_LkpFK5t&#*>h{%rvqrfm5v{<#tNDJGsN$t0VJ?}qSOD9dZ5cbA_Sm|u>1 zbo0i5&gB5ZFE_qK-FaKj%5Y91^TX9(NSs#56V`jnsbf=XUcP43F3tAr?zElLnT5&6 z3tcIZ^OH^^<6x&;P5RFR0Qr^S|L<;a_T1b59-c^Yeqi|Tq2dYPzm))n&M@%)cQ?53 zzc2jH4*&Cq|HZ}s@&i2&{4Y=buN?fZ9Q>~w{I4ARe^m~OqW}Qoe6upg_PJQKjF7HO zHMP@XYtWI*cZt)btyU*4YbUdlG3&|d-+MUKvA0c3wz?#OR_0gU2Y>c@9PfsXT$cc| zS?Pqc35lVsz;PFa9#}B33635qXZxgPo%y}FeH!?~eJrH(ZmuQ41ca(hJRR;nvS?Tk_Po74Tuw6Pm&YFgj; zuVE*HcDfk>#jck=FuWBV?TDi-9DRGd-WF0q!C8=^`s0fg@jrQCWt`q+x2EB5U2zx; znFJC(O=Gs4+Zb#^LCeoTVYMSZhYKeGfz%ON_TH7>-;uXe?-4Rt2#{0#Yf3=^WA;0k zw%pV@v&lN*3xv|i ztYe!7yz)Lt12x=+sMq90V?@>6E`UO=DPnfrhr$_LY-hthaTvMaTumh0mPazheH+neG6e~4Z!=EzDBRR>#7 zcC0m^z=3E6MUGyn(M1MDari&~nsUT-s@!V$g<2b;``=!bB3y_@x`t~#1v0oX343_> z@A=diYh!AgR;tVpb7xN!-R7*W4x`~V);E~|&*QJIkVh%Td9&H3id*&A5 zxokY@2TcMY2jntx$7S(Ge?-~$OKOZgJ{X-Wi5WBaAS~;(n0^=%{V=yTuGLB9?lF8okst^O-5f2kGgG?= zPzNv>*s1PXd3DQ*JHPnGT49#UWLi+r>vqOlP?>gp(y?UW^=&BBm;}?~2BnpavI^v? zMKG9ZF0+M}9nb9^!rfmVcH^%S`oP!Mk0+F7x_}YG0W-&A`C^`p={QUA+F>*_(Ib=k zq1NXL+PXm}RdJ_Cc~ie;(L4qB7DhTHAW#2dP9PzAqDd@|7C5iP=Vpjbd3!2;_?TA@k00R^|#jNeaXFJVmE8KA>AbnBv9%O#R)elua*Ih=|R-S za0bj7En^3&ZE@_QNO52CaE$R2Hz~QS1*HtH+y3lXw)`VEc_8>kty+C{HAj&#PXeYb z&IZN(_T-q+bA~aw{L=;!c`7JWj!PWP2HT-Vjn5`*6<7nKx%~Hl}6-&WbChIcX6EX zo1e>t`;6#7W_b%j*~`Wvj;=pq_1L2{^edfF^Y(+XPTS3%XrGkKXO@#Sl{bYBmS6a9 zJeN!E55(#isQL9Ar=%StpyVqQMls7GOht;Mm2>@Bw}s za5i7Unw&1XhHHfC9YA`O*QIOYaMoDGcT{8E z$#Niz-YlN!a91kOg zv-aH_THKH38n;Kiz3o9PX03g&M!E8->_AaBkv*Z1uY^pf0LmNfjapG6>o7kwzQHmJ zpK})dsBxSW$HVrt4p+DooH7`9T;zv)&3H%GT^`uz`9&jCz*Wz{MN<|VQUUdT30Kv$?+E}pRJ^H1!alLz?E{dqC# zv-}a}4mR%+!C}5OW>7C9-H+>K)azVMiG^HAw?^eAZb&9H14w78ISADbO2)7`yZK(l zYki6O3Q74?}a0nB6mvEAp0q;=&3t3K9$``TrY;>4?&!KBjb z-iha|h=NE$W(-`AWMiiOcsELmlc|ry@(&Q8FMZBhhq*1AQDG>=r=}RIk71;$z=Ao4 z3H(*$=wlFjAd;t6v zCt-}68-F`|E1m16LZPfL=E3^tp`@ilK-~=TM$o+P_Lhnqmv?N<#<8aOfz?lpvJMNj zh4xphVj*d9yW9ywkzYSdWttroBY%Vb%7M!yPIE=xxcJ+I4hwd_TAjy+@xJc_PNIJ! z&SasuN#vL&@(k$nsO^(P0i#4zR)u7juW2Iu>eBF(ymO>-Z!I3(U9&DnLct%1|0JR>}toY?|QPc0W9+D-Y9JZCQ zeaQ?u`lP>(qd zXcRiQ(rp7lUZrjxo$o$uatx{CsIp-#cZ+drqF41^tuPO3Ms~s#2bbT8d>J|TmFj8B zLJcX5G{WmOqz{Ih4qsLY8wM0BxADZzh-hH+z9pF|Vv-5kn@^v+Ns=yqHpw3=%ZQu% zrmBN|IM>?DSY%nnQYDr4%lm!euE}ts0@-YX(N(+N`1sNCz|q^_8e2@UckkOmQ{5P* zK5@FER2%!P&rzq!Up;n%dL>m-j7^?51|M16`$|3F22R+1Dr5V1*C?5){-na1s_pvP znH)WAACmH8alS!gaW65krLrz7@OJL1SrE-y8>w%H=c{ada-4CYF74%A<3$e}i z-Tuug-UP`^l~tnW^mowtBoyNgPrCz8u%1-ivXWWxTN~&5z_h-J-J3&KL(28aMk{3V zz`lBlcWP+C(bQ}%t2y|;=Gz4~){1l*WCycM#0WWdq4xpa;%DxED6&V?p;y5OT-_}r zalJsmF6o_JUrb|>dLWAZJ6gC(n!Q}C;m?I~^qX;S$J&bB54;p<0URu zMS)?Kpz`s_dpm}|_^>i-Bm4;Ta`?D!`!XC&cf}5Omhu(8Gbg_OR_6^1?fyx+B^21L zxJzD+aFeyKvk9I5cN_g9mRyqcc8offe)@`C(ftkL=&|(qA~Tlacvq((*J^T9d zoFJhG>-jZP>M4x>0k3d{qu~!eMrbERiCM6&JCMrEtN_*s^G!V1Z z!v@_Fz-OJn0Zyizp@21b4OX&hX5^HcWFP&O55I6kS(Y%0%P9|jqha!S*!MYGg_da( zsyAk;^=(?%2uVUQM4m;5ID+H8`J(3jMBC836Yc0g+nr^pqyuEssQ199I;8+#oqs+& z%9~UgOQwt%%O0ect`{?W%8_>(x#HROG!K1d6J?a@dHsb~(Kbk;UlQW*qg))}0wp-) zs`P0W%0x&eZhsJna1iF2KMwEatt@%d&&~Fmj~>_FY=m`J1XJDM`dagopUg4|RmS#H zGHF%hPrE-Bb>o9SYJl9W3%eY#s{|HE=jDb%!dV4zv2scP^=EJlaOh}46=?RYv3UDF z5pQx48}Audic9}2+cq=7R}!OC$i_~%CA$PrHl;`Xmm%~ zK1ud)S^{P3t>}f>ni|(;s;qx_iuzHfT2>Xb_yu!8$?s58zGE!UTKK}E^p6wsq|W+b zRC5f*nG%?)9!M&~S(ZIIKQ1JWGjWbD7ph)Bbe+^jy`2~fn63|HUJ{ZkKb&5a-qHHX z+wY5`_iw^e1o@q{DTkxK$x7x)T-0j4pZ7na-vKN`0T ztSEc@;7<4?+Q!t4wVVSp6BwB|VY&kOOqfqx)#5gZWFU+VTn2KoIRuhLP+_*ZX7{rKi%yUbWjWXWaFhIy}9DoQ*;p9OW*Za%v+biY|Oh zG{01bgO;+rsfrm}-UP?PJ%7p(AI0l_mIB6|8nU2geCOM~A1-yvPbmi>L)W+8o?>@q z4(geOelQEZzj0bEe|i#lxCRcSk}-yCOvM9qM<_I^1MThE@^)$4$c)Gi-zXd~DBLEO z-QTN{E*FUwG8i@$wlF@|fVJiO%UMlTcNpfV+ol&88e})@#@M_3$X>T%W9&E6p#aM% zTO^aWaPOEv8QDLo$d$B*^Pm-FoJoJU0v>9!%^9^ui8lqPduss4i0W^)J|2897C!iJ zTZ{Firc?SPH!bjJn2vyXwC{DN4wz5adzPJYHS7|OER4ME7)^w3pD(w~4$ho*d^0>8 z$dK1qI&^LYre0RpOW*SC4`4N|nCpp9Vp(mdn}{nfdS=w^&=FT;d(ZI&_lLxo|F(IV z3`gaGBFfMkgOoYmiEM8(3+ahui$xJNC)kD+&$k^h#4>}4=5I178g#4q{@4=y4-99m zW;V{;k3BeuQRT9*tEiX}Y{cj+J*`5$0$TF9Si1M43qg6o2*1lGorZsiCC5tB;nwnn z=FL7pA~bJ4_I2$@GdX(XU4n8;syr+vZEr!KfC=|mu`DyGkwe)K=8H!~cZugD+54n; zpn??3!+*o!m50A9eFzd3R`A7DxZBBT*Z`+QopAgkjl{8K(Y};%>B>F-gD*3sol=k6 zdme`M{YLB zBA7+u(Ev&NrsDYqHvyEOKCt{L>N31808-3R9_&Buny2AH;>xjcSca9D8rs z@3z*XQaIx?OuWz183we@Vark!uXt61wl}dv%R5)P^NS6d^-B?5TkvJP-ojbkiV2k& z<`_jP9D1ouGd;N`8I)@Ad8<8baekJ^tbv2~{0J=XJL$f9Zz&X;<0 zbpLwzt7}5xG@m1M_b&RDO-ymoRSe@Y3{mB5x{hKcBd`Q04(W<(M9VmK<(bd8e3sCm zY>3zN;4u;;S^6SSC)KsSqs*#PE=aX~Oi?AytN7;omao?im#nM|T{AJn*>5lV0mhH$ z#fhusWZSnxG;bcayNlM{MXOb_{$e7IGPxRcdiV=KxZ#y-CT11&zFa&Ob~@$fCHs+} zRBX`W>@MOfi~$L~e7|lC7~QBCwpXiJ+!f(CR!neTa)=_V$~LM7$?r(a!b=Znvqw(t z-eB(%sVdV(1pY}Ll=TjI2#YiWhBf}a|NYiQDv5~5xAuClwPmsIR+401_q3#yfv*F0 zMLtoM8T)3{b>`F3!o(m>%*4vG>@y@xo!~WCm&qBS{qkMJNZ=_2b0WtyHShU4_H}iv z)dm8JpIls1chBceHIBP}FGBFd?Ak-$jbyG{r&omBA;r6yd-o9w-6##T5L%CbdIb}e zJ4cYe8mM@Y%3xM!p2gf3NXHz;EY!|1IuB&}T{Pb#ADFg2(R+Q0(@pcpn%SagjS_Z0 zEuvp}=n$}$D^;`7Tk>WN<&}k*X+&}qvGot;(H3mTP(fwt(2as{khH{Q{9TsV<-ZZL2vwv}x|gnKEdk+rhMynF7}BFPiD11(2dOw}a{_dm2S-npR_F_ynuNlgN)1m$}sB7XkQt+P9;-5GR_RfXK zfo}BNSnS8p5m#WK+TfAGJ=C$3DAg|xB80BT7d<5p;}TpehQe)5v(A+vV@tra1!t3=(pxUQ^^rT z;2p!K#zW7RRpl9#ismHYO)0NIypg}g*YIm-?7DdHhDPC(cxAsqZ)X$Q*;q`% z`l+Iu_|O-U4J29RO732v|t!Mw@xT?LehSDs&7cc`baqN(ol z@aN^tysHhRYJRVmW~bufHl3F8G;#ia>phQvpiU(XlVmaT%4wm}Q>(nrU_Uhr)-*D3 zB%965@-!aOlaplvbMbz7&3)Jv!A}YfX`aHC@AabzI30Zjk58`hE?~1a51l53ALdSp zS*QD2k~aQ2?Rb49PaaLcTf=8c>{=IXM#JZYz>=&#Ufk*HmM{f#9o*$mx`~+$6QqDu5m}9u) zdTRKrgt9bHRtgwc_U5;ALk^0cWK7c9K02g$)CtP3Kd`7(onbqN$%F<4efJi6i(`6x zMdf|ryFA}|v)S@@9+y`>GpAG`p%+QNO#H}OKfm*XkW_jLKCjNi!M_8S}d5Px0K z!s>tGS{09%Xq|47ZdJnOww!K4Zo{fC#C5~;8$7mtQiEK)Npn|_(=~|_1t7;D6REU- z$gbKQsV&fh3i4yu*i{PoH`~5)a)e96E}SWsvv7rW>T@q{Z@W#CRBA z$@n$%WZ>H&1|>rRZ*ycfqDeCP8AgVOjvOu_{SPvYB=7E*m@9E z@%@p6j;qf%sLVlMT9w4&XGxmCTWS(_35U2VmaT1-1y$n-gU8e1cjc$T6no_W9@ z>f-9bUO(h%BbQL=i|5E4F#Pk7Cn@16FvTszxYZcaa}$s(@-&qs1B8H&FB0Kbm&BUL;wWIkIe^ISlN(<1RVb=PJl{+&DV_dacK`5&aKg*;VoODgCqI^+H(rzg)`UL(~n{Q#M?#+T@8 z>5Z|oqZ7~H5;Y)#amva8l(5XCC3PJL@frY1z3e7V#YCK(Uav*z;Vwi_#`Nel{RhOns z<<9ivt$YLHdV8x5um>f}8tzd2FBzVT7?9A^sz`or79~flA!`7k4yk%$QTtQ1&D`aa zx1#QTjpZ9d^w>HXIZdrH-VDx@IL{X(U9~Az<-W?LiHU^bjTtccn^Npt%L<9r{7rLS z&-ZVgJ8No8M;A%v!*^A9WVl>uad`@TqUc9?B+CRqnh`hgI-yXyTrWbbm<{Q+YPgX&-!4;;v_r84VFZmTo-bqnS=h`eZ18Bkf}k} z)e$puRwdUZ%7V+O0--h-k^ip!$_kv3h3kun$l{N}csUQuebgp@&R>(?0|>Shx9H-QIE z@AQ%D(j{-wRh?AU#1p)yO_Y}5zn=dJK-A zY7+!M`nA1Tl$$L%w(RK3R%Xd_5s-chaCZ3%S*$2TLg!TsJ1M>qS9XUt!hpEFsK?ZPh{tJ+@K;88|JA&3ax;+Z zous^-YBt*l83^YWL*FX7E`IB!Y-OfTU5-0K!_GseA{m-X$gylQmh2_C%D<7SoxO8S z)z;GNa+y0{oLgD@V-B(kp*x zuvS3vo&RFUbhr^!G`s4O$8PORDt1Xodc;XJurLZSIlsRJ{zyY>A$IHj+(SnadfJ#5 z9l6b8VsNi=bJY>S?}JHR4}NPy!1MT4Iotc+uH?=ISP9*oo2)krx^iO&oUB=)q4OR# zbD{;^%g(^L_(^RfN(9h~I1tq?X==LueWqKZG2zyognL*iB-;o*@72vW_$q5`Jur<& z4R-ACO({L;!Of5@c|Ji7h=F-t^ZgNy(J!d~5bA{;tJ-e7<_>q*y2M8Bk_TT315wD~ zR{gWP#g|BJVaY<$1lLnJvp2~WkMw$u2Q;_W9_}H5qdF{&f9VR6rPd40BNxTk ztgIa1tipots(6701izyi*S~9-Ia=Uiut%T554JtZB z|72UMpg|IQOAz#DF53a;R&^8BOVlEO)6fd7b~T;H)z;2mpw9ZBLCv~LGpwmG;k}P$ zN{(sAkW{_f0noR9zg;;!jjPHsVDq2Tk~@b(g4H+wgq_9(`zoQhU`zSGMgE9qRMV<{ z!?SZ(QS2XJNOud3!jTIiiNoc4qf>Y**!0bsj{eqI(5&y$e@t?ynpZ7-EF#^Yn1k#L zKtdlXVt8F?+dPBSh!q?WVFOs52A$G+SNK?%HLhX3`izevs)ds}tr(>`DePe;j*rEwI zYmrb>j*y7qFi4hPhuLbbh$O!pF57gur@!g>#n#}t79{8Ko}_&@gSh-TLdOPHNX=T% zIc(}rx;z9(2|PeEPd{2~VMWy#F;5(U zdPkN+kw(f|hUs2M$`(M35mFz&pS%BZ9NqjyD4#1-u3F$G@4t9@dU z&3s?TZSbQ>$R6IbCebh zAn?Qvx@#b}_ZmH7yiYty_>lFg-p$$z+j;pk4}wFfR3Puy9|-X@KZwADb@ z3!*$MKm@Bjz0lmNfopJrYv~xD{kFq%v?}g(($E4A@rF}|uRj)tTCO*TkMpA+XiMoZ zTO=~uC5ouDF7Y*%1$Rf`pFvW7G|Myf@wu{KPfw_ffRog4z`qb1j1t^;!Q{?ZxQGt! zTjrekph2@YSCfH5w+!$mkjqF-L9CDpAD=Q%|$s%MO^Hy_e{RS~SVS8?FcMIX>@SUB~Juv0UHk z9Z>lei85MI{F^+Q`I_Ry1vr)`R%M?3Q1pP#8bqz{AS&IBingO%5R$N$NxTQyS4U(7 z;(eS(lM-Rw2yaz}Ti)AmP}6fb}BRgnJ9X>4=>%5omAS9M^e4(Q5!^_L~O zTFjzcMZO`f;Fb|$#&lgSo*Ai-I9A#BGcy_NrypALdGi-;H*3$r=msTmRI4)hJ(ADt z@~0ipAz=8+Q^Jb?)vW5&0_K5c>rL`)QSL|Q2NxFk zU}k9_19x)r9jiBXDtb#iuZvg4)c!F$zV_?jtIYWjY)hSkVRrnbY=xe5Yu|r~T`Hn36o+E(lnG{v{1N#OA^EFGC26iGTILkDsTHGUGT1dl z4sN1jzW3yeN~(0j=hFtw^yt&mCmu}bI$zozzE?bkLi&&Nw^#&0$#|Oo@m6c2Q%#!v zNTJ!G6W{onLt;p%2V;pEVlA_IzCD2vV8trg2e29t%}cu_k{MKLWPP_9sbCyzm+GE$ zuEm=M&2I>!c}A~v{ndgHkA(dblsDaKXh|})T;9ndHGufC`Gm>nw(+El$rFWoq}w7s&` zXzyQruqM$+1Nbo|$< zpuCZ=9k3^YuuXIQvISh0sG4vZ4ZmKvu(9FkCAb{DYV*LplXwf9$wWTuudqL|pGj?C zk_nhlDpelc4BS#w^Grw?l&w>`5q>|7M`3UJ-uLZn$B-(H8k3C}sy zf?upM@>7r!C*s}Xn9$pYX4D0^s)2|hnxFyoOcy2g603tw zXQL#Z8NF0(Me)s4V!2F*=BiUy$qB~~WWL)UBsIZHQ7k;6mCt(WV0L4!oetEN)>qim ze11xS^C3R>s<$3Xkr*$xvsIXKun-OdbxzHG($k5ULZFhPAxYhght`JjBF282h~)NV{wGhiJyAghTt#^IChPh=5_0by{b}JK!hj z$NO$odo2+=`FCQ4aTk)q{FO+)sZ+JAcIuI6sNsLbOlE-e8J4`XR zQ<>>e2_HVDzIiWsSBURBCx5CY*w?EhkhS`jSiErnkj2MZo~SgzaMmQ@O7EzNFEQ}( zN>jESs_m>hud2grNz8TMVbyZ&rU?9Otu}H%m6t97$z`%Euye!El01(iuYgWzSP;C+ zBureb0(Tb8HjdUf`(c<95NEp#OmBc*?;2qJE3~$|QbKJLIywG*yext-Qav^&vGGY* zryV^wT$ff85t<4%tXP+28vLVKEuDEa9rv>DT{d#IXjOc@`|Q-J>+Z!Bi?!RNmx2v@W*yjo58RgLs#c5pW9P*PN2fw!UU1X3(~w!zx0?hZ zay^y1!y&KxyS}9-MPR!oA~U`iQ$8+K{r-mh7hNRi)l}_Dg2Kng#(R5x4^Nps9z^*& zL#J;xT8YR9Eq(shu+hLspQZHbB_X6TE{*8(gs7So*b441bTu?h5M*UN4f`a0V!4Y#$#DugT%CA&?skLE4{b)!& z6$5{c?C~JGr&0NFV(UDyd9m3&8pogCeZSp1`DF669Zx5Y)}t~z$E*)NHgV6)O2ngQ z2I9o687nxT5l;?9JN(xlEy&qkdd-D5)r*nr&YV~%_;(_de5bFz+p zVAsO(A%HGtl{LxBGBI8}W1>M`Af4S@ye*%+)2sFv>aXS9x_^3SNv|4|kye(Ew5**t zu_&56;^fD?aH(wi8D>BZEi9r4a|Q6g2#AJ1>K=F7(K@rJk^L+E_3|yI0#?GVm?4(d z_F0V-34Zxr+xAOyRY&u6i~m%s4j%Khk84Pw0MaY-k; zdB{QNWI<_#TQ97D0IePa&42ZZ?F4098Y?(KAGH@tgxfOQG(6VWp%0%li|*>IrAPPR&dQ^FqfCOgNhRqk?mb5xZC;q{%=qTU$EZWzi^ z4IqvSe#kVVprB(L4W$e)RI2dT$Sdo*ntWTjF;n~JeOEaagpE1R;3 zu>i&h_V;K1*Bgk{gzVm-cM8rgBiv;h?S#SJ!8s-nqWw6$)_Iey*MyPwF%%zX_e#N( zU3G2nQ?lcJa|PE-b?w;U8{R0!-y8GX3mOD2)krb8`Te1raZ;^4;M0@a_r;!85yNb`mAx_Lo$qEMZ48OYjxRrJrn1{#|Q2e zFzH-uRk}y&@Y+IhEW=CJFxK(jtKO#1#o@MTIl#Gt*b9|<=P}Mr-s#5QiFV2u_mX3h zwbo<_Eo4&Y;FIA-T%g;j*TJdtJv&0{yX7QENP>hTWdNML&YzK72dLH$s3*b`z;rwqDQHoZG76%G5pHFue1bb}ZP^qz~X(r-6&R zDI0OAZdJOI`wZ1`C#gvExHe_N>lIz*_Y}E*2@M6*9?V8>XK@6-01RE>diy(f%{<>@ zuH|K&y^R0@d8Dtj@=P(PA?uUyemY->v~|Yu2W7>5|MN40P_Jxw$%zs%LjuNO%{BU! zHSbSj!2u~cI3U;{2GY-FvbD=Fa#C|?{AY9eQ$jVdRuc4VPa9E8UQcYk!FNbmEbl-d zu+9IZBR`H-Xbbs|xcg6uSWD8X7Fr@41`amDg8d-CM`#6|(kV!-vT=k@=cW#l;~NG~ zo#(~~)GZ}>jx;X=Z40l^Oq`Fl9bQlCZGzf{@lp9`}gy_Q<49#N+I+ z?r}lueZl8+dQNCiM7@t&w4=&>1M72de}}K4*YP_VslL1wp%%g2$qr3lE`91zQCw0y zJU&rNR8u`F`~p7;_gzi{NA_HYLljQT?ANJUf4`S2@(7f`WyiBWDug?Pq%>)xY9u(i zW3(=N4*p9mSp6f7_~jln7ai|HL`?~c?vO&tDDD>CTuvoVNN#02qxg=eA{U(l1u}n@ z8Bl^Y9+_26Gy3uRxz`b2>4(iogwMk8022A5(eJP1p=RG_6e?YxZ`iY@aD1XiIbJS6 z_J(}lwrbF-5RI-Kx;~Zb5;n3lJ}E}tsOhFa6QYY0%)(<$P5Kn6UT=4o`zw%4WrDo< z(&(pWT-r_s3~@ehHCe8y!D^3pDMD9Z$sc_pkfQjL%{kA1mu~NDJbcLvx|R?o+n|Zw z0?ue%L}20Ifv6IiDhDd7bWQLk_{(Zh%>GtFmgdkedt6Mymq|Yb_Ov3FiMM(eNN}P*A5psqt@dUqQ_i84+p}(6Nj%*y zH^ao^gzfMdV!P(ij2|jCJtKr21>Rn|ciFr>=A^7+h=gM5Pn^{41H~VzC4{mQb=(f? z7d}rp{6mGM)BCca@3d>CIH`L8ZU)}`lmPNRAF^V^oIbHKOi?~mt}m@Py_DK%W`3o>2E1Ro_zMoVk`fWr_bL{p7{HpQ%u6RZ^lsjao60iStQa-x;={FA#si z+fcS_w-Kny5+UTs(43f5CbFa~9YNkQ^mpk|qbpE{XT+P*`&cb4-b_9=ogjB)4gTCc znS~e1DaguH?0s({N5{8HtQ(w@;WH9$F~;?}uYKwXlK1A#$O&uRgbcKABHo8k_pBmN zl2o*6hbw8PbxKyRgDyx;h^KwD?eq+x0`ee@SZNSM{s({Wyb}bJ={L4R_H5A%d?B*E zSu)=0d%;97lGNKhJ!89RsonawEc{&R*Ygx zxv97=x`wD`!w;J?8*zCZ%ig&5#w@Jh>9! zzPfaQh$(;MW=*gDQn{*Hf)Uv#x+>(ay2-Jot&ggr{Jv8%mSOf}&AfMoNQn7C*r2h_ z_%Uuu_SdYHMh~4w_C2A|>+i4pxu#zFQKU99W~v7kE8h3(thK0$amy*hT@=+JUKVRF0h-4{ zz7578bJX%ee{<|=y)iCaNznNAQ_Qu$KZBlm)yT-Fz+{KHUxq?2h_pM#8D#@^oYahTLM+Ah=p5TaN`>|o zg9^LPw!BZP{K-`r#o5A}+t^n{!VXhaTOkWS&SPpQ^KVFH(TT7Nzh{Hyek^rWT$0}* zJ<|(B8)+;ZS^4atgrky)3FLl4W%Z=t_-Z-@dHfY(z( zT=(q;*r4&c9KSC2*N?r*awEQ+JdLcUg-?krDvKnD5R;9Z?0`dbJ!KT~`jK#nC&a=B ze-ND9jB(wMlz!!eJzCEo#Myv-_+dyp_9|o8e%qBTJ*NHY&pu=A;N{cWD3oE*n>5{0 zx6kk@6+gb9^EmNL+T(KlD}S8nyYqL$9;S0njQk5+R4`m~@Of&zw0Q*8t)}*V>Y?U4 z?QlB>7o`$7Q*<;6=IQNBe0M2>Pm{?I97_8G*+8$J7l9QgJk7`=}VdpDM%Qvk(|LJ4ercSqG-Sv+yZKE3IR%wxT3rbi) zvr5prI?BEuW-qjUgiqBJ>$zj1T%c zRVL-d#YO_itEv%o`c`VYi4asHa7iWZRR{WPq3wba@?5K39?4JBX!w!Y93NCBD{daE$vA7uSb^*f6goLV zFNF*ra&%Pis2TCUH2FrmZNEb8cOD*7c{L|i9B=$4;l9*^29&&OoW&kFIQiD`$rCAM z68e3YE>7j`L@IAHzUAkx1(75a{+0Lk5$)U@w>4F5d@|rtAb5 zZse5s{b#j&NqPq3XYp|1l?-w(3-Kf6Wg~d-06qJrZ+la5!86Q z#pboYcTiziZV~ERli7a8~FdvvL*p+u;!s#;~J`uvAr&1xmKuGsT?cdl~*T z?Bzg}LYt_L`EMU_Pi}g`gV*7+KaN|W{U(pn!l&{Um8DzPL!9#8iv%7h&`=xnF&d+%P>)WS4TvLK- zTLq+kR+hcr4n-JQNaa9&7?l;3wEl3n^c32nA9nr356WwljGlWj&Pv>c;E<_x8&^W@ z8_Y-*D!m$h)X3wb$vmR}Bigkv887pG$s!#@!8)vHcyy{28aa=;z=a>#dmik}Jht=@ znyjz%v_*U}Wtz@kaO&AI5om*a}l;=-9yhf-*66+V9r}#0%YC zR(ZT$X@~W58y*Xurzb0T1nYf+r#_<{_2^v7;Jx8ble^8G@(DiSMS}1&80jx|8?8kb zf$thR8v$Q@A7zzX6;wJlhmHQ23>Rerhg?06c_8&O!mftX@|@Fc@WprY!&9XwUD&gA z+&-$rGqnJY+~RUjYlDB!h~{$#8hO)?nQKFNo2Xau0^a|}(^oh&^?zX#B8r5HfJ%K0 zP)QN#QV|tNg;65~M1g^H4ln>|m68VOjWKGB9@5exqee)_7(K=q8}G&6`@Vm`?!EiO z`JD5d=XpMR^~oYfu~TFbKbP644RXi4T*aPSWMpaPQbw^H%be)SX1w7iu-s+EBQJNF zP&kW!JaoRCo$%A^>h-fHqTQm)^@JDLnG`g}cid-*peVjqAj4dGeR=Bc%aJ!wcJXHN zG)l@hY;Xab0FA2R=%bz%v(QQMRl*#*)=bWbSQdWt@ee*LKGA+w*6@r=SHG>*pOM4B zV`y+9GX1#d6ZhN`)~bsW5xn1Y;kvDu89{mN4@S6$G8LZps!#*(U6Tg{P$=LmZ{e{! zK_dM;%t5MtD)V@Ofg1*-kzn=>1v|$HpH;6DJ$LPPePLJgPp32f@zi=WdoVLDE8nZg zlhI3^?zBh&`=$x5Te4Nte+FQiI)OgOy|6JhKND+X9JrNW;)Q)L+w z&*;DJ4by+i>&^Aiq5v&46cu_jubITrG(p)K6&~m*rKs4>A#Y}QPTdw6a1nQ%+$qj^ zRDvv96M+XwKQR76y!uz$`)Dw2p-a+5GDC58FwK5?ooTx^iDYbqSRvz+8Z5Wj7;SGt z`zFn-Ui@hosNeJOW4hefN8>jS7rJfu{{7lDI!3NnaEv$AHE}%0xa1cswhw((EGzTw zqsJf~UWwZ>j}{F;Pb&|;7O&YwTevlUL(;+;gQ1Hr32LJG-EP0>2@F1G8l^rj#o<1K zW~EDcr}xuKGmn(~+&9#Oj65iletDp!yg594%%%%e!cvGBs1vUp8y$1;x&L;Jl4p>d z?h!soqaK6A`^tZmlKA%H%cJBj595oc4F`iy|0wXKP_*Tj;w!M#yk}uQX%^;0l@G-X zjQ;}XBiS+QkI$K{!z@9gF9OuWFl6$hZK>z34ucgLfy;ML z&HbsRuKV+JC4prLby-s{-%RB=b=%L-eDS+Tr#3zv=zN>h#~gKH(=1h0H!^5(xpDMoj>|P`FGFW=u9QY| z@9C-bG!~k2g-ObO;@@|rre@=82pnw6d`K;(EOw$=Im@%7>4PKeP+ttz`~U@{-jPLz zibD0rc8S41VMu+?NEM*F=QFO5CVg))1W zXuEbE32S@P150eo6fveM)QrhCxIye}#`RxUZ6dHX7FR6mLDVTfZAI#SwQGN(V4zdx zispdgnZs^amM8y6XX>1C6gNcfilX*sBYb8I>^sKq$n_ z;d4JYmmIB8ld%u({!ZL~QlVR;%s&yNv{AX4Y(bhf(uY**jL|GCKqtG=n(eMWJ;*ZH z9^OQn304COVB0*-0n$v`*o~cJzheoBl|mT>-J9Nv6R){X8!Ymmn);Lbg$cheHpOPm zF<4=$MxSW8n2ZgMVf=T3yGC!w1EEq4EO zszz@2=5w4E=F`k!8DcPABB1gB>Qgy%b?T`8iSq!&6%*t1vGSc-1U?vhYW2q)?l#l9 z7{EL3j7GgO4QMnLo*hk}kGi=SDHVGhr+lW#AF-YXT{1^-Dn+t%sSz zeSl2AXQ0+N^ScEr@9r|!wB2pXFD4HMhEnT_j32i^=6KmO(rS_BJfl|rQ`bJtTrKt& zy$`a^z6vaxIQULo-U*REKI?^~>*O0Rg_Tb>eh0*Wo{k3-Z+Ok~3m->C!WC35^|XOQ zlV*lUDeS{G8SercR5${>+)HH-ruPxIgsN5LTJ(#=PFYRqB1Y~yI6{z^VI;3a_|8ap z#=%kO9o)0Re+>g#HwC1H@^3fnshu{0`O;`ci|;MV`OP@a?!KpO*?NDAk>wXI;sFbx zf_;p!Rd4^Cdf=MNballBs2P5q=6%LBKH`9L?Dz{mR6PD}x|5rhdEv!-ZN1mAQ{9fZ zo5qRvg`dc~;RMNz&gwtMP+It`4=GNv`#%*hDHd6{FM%uAL^b}e>GRXLX&lSXV=-8- zAFiw?Su+3-Q$@8|l;Tp=>nnc~I^5tlC{)E(qvWWja$A89N^Y%O99^~X8_xV9)wg-# z%1w^mIK`V6PNcW`$zeyHE!vuF9FA}bc>KMsFdIE}if9MUAq)spY9lkD`=*NVAp%dB zF0a^tf^MZ256G{4$^qgY2se=c%^wPeq?#%gJ%_~E-8AxuQt=V8-2C8?WT+g4TY`+( z0GdLh#^s#1L=xOUz%_J+2U`Yza%Mc|G+&rytw+BnSIq^H_PRR9VUI0BmxE<3q;r-O zp>S?-XE*3VCipPy(eEU}fRtfAwKkOA^x_7O$o!LTSB(Wh_3%bBcBTE?4~G@A?}qX; zUxENyJ{MhM6wBSg)^(+I5TER;kDIn}D#FcwJdTCJgN`_Mj#}EUErda0%%^0seuzm$ zb16tyAEcg7jbZuG?xp&d92FhuLEG9FMmm9l3;QpN0O?fCuKz>!!xiLi4secw0^5GP z6tT%R6!?K%%o%b@m3UL%qr<1XDB2vZH=*sV_BS|>Q?J+bCNp$r$w=QSffe;!w6^bV zc8_r&k*MKi#?cnpS{m8y-!OBP(I^r8I>-{(gE4g8|BWzmDE|ktWo$w)Fs_`Ww*p>$ zGNXpxLj?#L!^niX)t#s5GQ&*ZMsHb&Zg{5xt=Hlap0hx)`ej{4|y+g`PCOsc; zK#9%_5q3TZfz-{IV!=d3Q7Da{g{rl~0ZQ>xtIH+%;Z}*G*f1NDoO@}%0Kw6k{E^wy zO`kMnndRCVE7!nRYaYs^n*6V&MNun?uD*yJ7N0;J0zp@v%K6Fal=o*FVD-9$+0Zve z(Xs9o0rp0~a;6tqC!d4hNgUrYI-5q=$|LFJSS@!5{e8Y!y`kgh=e}{AC3Llmo7e2( z!>~<1X%&s^U{O9w_{^aee%^JCv_@=2-sATN745yh$6mif7>K$1MP!5D{F61M1{eJ9 z^SV#llcC~Y@f(S1$~)R;fOctz*I=Dr?cU)^4KL97C*A!qt^7t|?>on@Tl*6)=>}*} z7d2Q>H{ph;=_pGW zGNIbrB7)FACz{sef&PlZDnQS9UPzJ#iSHA~dqaQEwUA;3&iBtw6z?vq)*dJQV95ww zLmuC?p))RF=tT9jFS=#f%((EOwJNNTO3q%je|a}I$(4}PDAXhv0?D`%n|uF&q7&-puwz8 z%T?^rOiI>|^@+gBmpzr|s%Lxhu*SpNk&fG`AFsMoCBv6zaPQ4+pHQDb+%CDAYk%KZ z-g)VTu7OA!euiqua-LN$C_S96U)15{_@=82v?mmNM{zs92KLF903(je9gCtrg|9er zq2+h5))*ZF15hB;OCBJrSl|^VJt?z0BRZp;fAbAdEXwW+&uUL-jt^XO7+Ci%E4`?U zoK0Snw2YN=u1e+eZBg7%vTBSLR1M;ud96BwVrw44Tg*4VftJjDLrkOneb@04zC)mg zQEM(YqHvdGCcPiIQL;@SF*rW*dkZ)5cXgKX@#UN|lWqqVRa-NNEsDCIp-c0k7Dg8+KGJuKOiaLhDK-QGq?=~h6Hi1N5r2r-l z7!I`bpmxVlW`f>&mbv?rF@zok!KF!}Ft?J{J#4i(>k!M=GOzJ~Cp-b0w3Pyf9gY_{ za#i>7Mz(6}-JA1x(V=Tw`-u|~LX*Lx)F*%h$n~3UhU|8x^3mZkycH45+01MOZ8}7C zd5U_HQTM(nk4j}t$#OI|_q@PD-@_HZTr2{JOM|ws0~e3P(a$#IF-qBJ7q6_jc$jwX zDEUwZIY-F`<$Bz3!E=hFD&FKC9YAJ?wXRM`iXjNs$l^9@aJ9Me4<(bX)jU6PHca!_ zMp3>lgVVi2$1FC`XDh5+$xYXff(RclT~^D~tTuoT?WItqM)7t)2fMm8_y$nFYL@Wf zg4f%Y4Ul8niuNBjvCC>hBNP0Bq7X`~daWn6q0yq@OyjoO5h0A#N~<*Uf1oOz){jV# z_5P{T{v7m|SFV8q3&%?-`P3^JCIMZd0C?bZOTFw7W2II7RmC45)F{N`60DhK>Bv?h zb$m-H&QvQiljK@U`+$jLN7@`)MZ{-F%e9Yx7t=OpM^{4z{23~dvQhjML>7f(Jb2++ z4X0k8pCwS)@N7?7Q-VXWr=HqGq zbN~CWWWZNTk2~7{+~~?SzB_1q3vWObcxIY=rp~nZFcrOBN8!n`WvV(8s&i#KN!5=! zXj~1$NE{`>FzwvWs}pPWy0KI8W3U?N*J9p$5obK*iF!x06^1KF37)5l-kNbA+wiGY zJkxujG63|BhgdE6|8aJTZJdNBcfx@b*Z!x(70{Je_ZICvHVZ1 z@omm07Nm=XH=e=5G zEoR%4%gP#%hkF|&jFGBLP>ttW>Mu-ueeglT#QvRgPG6&z^_7#{tiqSGKOjpPw2Cgl zC(8X}y{9&lB_=gP^6kOdaCz1!**DofK+z`fqMaLV;XSSqVs8`ko=u^Y>qDb*Tdn7J z`+QUxtnErm)@HhF)MAV)6a+u4*QY7Ey5R%Ve%GG)$TrhmGD>@{c-CzA(2Bnf8e-|N z0(0Ne#A_99&}RSaB>uwqEN2-dQY$4TlJgpC8%EHRBK)nMhw%4}$H}hKPe*t=>a4_g zL40BD|A}S$*mYdKL06ZWaj$m7%t2oJ1?O{TsVUbuet@$+Y6h!w482nNvM!&C*-Ye81Om8(6iMp zoX|oL=O-fymuLnZn?Z5PZ%LFs!Cd)LmE_ldGSw^Plkd*erJy}Kdr~(VA^~w*)UKrJ za_9ZNceJO$tBn-{4%638>?8}uvml_I^~wPw_bwiNnUmZ}zkT0fpW42&wJpU67W{a# zYHz7K3wvJ7!1=2kI2#huTcPT=!Xw$ed#;^7_kyrWR347s-VyT7Inf2Me9>GSM?DJO zZ=2;0w^CYnY!Of!P%q1R#}BK2f;{$WMF;kv)R2Rv8O7Z`6Wpb^JOq^8|JE!|j|LYS z1D_jk%#J*XTSc!Z(qKUwKVSM-p<*?ay4xJhv;);5{MkW&(Ol2ZAnt+(luea*moTw9k)YvXrSn1MlOP1! z1g)WTqVDy-QBMxo`pKN{_$Ig`vTTuJDHI@Z?=Guy|5d5mhg%Let5taf-!}KD$?YX9 z@9#U@a8p5WG=*x9TrP`DC!&bL+F}JPlX*nvaW!Dui<)^?u}4) zuZde@QrhYgw-4uNd=avUh=dF}hoABUtS<)_VKoJxS?(GosBrM!=EXQ+%Qmpfqt}@a z8(p=ytW*}d35#h$V1ahrYnX_$q-#-`v!v3>IeZshi4`-F$0(RoxYYPp*vLVoBuYt= z`JB{pEUjNpw+yEaCTx!Enl!SIwF=SVC6s0%%^!3J5BNs<^tYUSD2%rM8h~0Ov zUgh)o?PPY%lk05#rP@>GpITJq*1N0kkx)#t=fher*{cYdqF&Z_J2$meo0y0J4HGlJ z&09*YH<}iP)qV*ILUjYPW8*W0R5K4cR!kDPa~XnOlDQ%v6G1v2+8qF$@OGg#6hkK( z$%c_VWqkHyS;z)pO;=}*4f?(Tc(*W=Jab}ou6ncu$H59?^lQB!J*H->*u<%=*%3~; ziXf2|zX#$?(}z?>Gy0m{h=CbeHzzs#S<+(&f5YxW3m5TsV?FY$2ky))9`$?Fn`}E9 zUbROBzfatufJcKUDb_Ppuler|P62z12R@hiA@@nlMQaV*Uj#CN8pnIH{mdITE^8|S zq)0qAP^rq5V?VimqWJ2UMEk{`1{&Gsz$v{beXBuNk+?U>vj8`6C~Pf$jNI-s6jE|o zOxQF|Z^uQk9J%3g51`6{!cT+Tc!BO(qzFyBthJ2JQxcfrcGw8byxm)FFE*p2^o~-D zY2`sj$whcWsm|ExE-_KI=5J4G#1PE|>Xl*vVql0Wwa#ld39qZHLefxh+%USK!bF+$S%6T;7P-j_OY}!F$|eH7Q7Lqy>RhAr#@l^P z+0Re@vn^G!%$eZUMK^BU3IeMgMFQ16TB<(jjTM>TI#*ewzYflc_xg$G*;foi#I_>6 zR*NeaiWMZ6I)^;q6$|(;&QY7Kd#fKZ9Wz#OMa92^+=+FqBMAe1L~nYd0vOfV`aJjl z9bGeCz*kZ<4KXMXux)J$wU#=9o%vjfFx3m%C-mD~tor2mPT@Zasw#kA^_i+_!LIC_ zQ(=F6K+s+ZrT>}v5ZI+QX~9Wq_3Esh(+!zJMJ<88g;`QXnyN74=~9vA?#F;8`1*yA z_Yy`ZdXp2z2c0MmAHMg?#-Zfe9IV>S-(4-y(kD#}A77K~ay1arQRNg(NYcuvBR$<+ ztSi>5=^RR2z6_2rBk|%gyfoyhRgD4_>DsBd`11JJHI3y`bG5DOeH7=d%sZH_#iPB%AxM zP0sPEv~}-D*H^gP8>8+!z^JvJf{h!-n@mS3H_%^R2=ZNfAXhXoa9WAUA5f(*WGkKg zB@jw56U68FVY-@<1MlwGG#C10J+)(Q7(!6AncC-LEK8gz{rW4pGyv5Q^d`%5Z8zZf zA#I4|XPu7erjj{poyd?=6Y)iD3-A7e?r%vBw%YF67*AgSAM2q<(6M@+qg2`e@|a|i z9JZ?~mH@X)3xo)TIbtTWuf{gLK{W^|uG6O%k2w4?T$Irb96=0F_%W~Dj@d0}`L`A% zJ&rS~q$$nd^JkLP7lvdwcv*fDrxeoxU zoP2Hn>z{jEl9yQfdkY<*UfI%E*$axU)Sqoy01I}h-yMX&_N6}nfK4N?!joYW^_{1z zJ>f*TUOTk<;mvg_{`4zNzqv}YEXu^SlxQUvuhro{yEO4piv!OyMNJIr4kn?qMvnD2 z{tUQzUXDfmy&Os{CrtmG*9)pIQA*f~+Au^(Tmz0I(n*YjIjSi?jkf;FUN`zwb?TNN z&9j7oLTrl3n=@eNFyGWsno2{U6#V zXXx)cZc;B077Hh{DIJ z94-3|+HNj1??c&kDW2t-@g9QIh`^yp?do*KsTJ>|Nyf74KsTc9JY!i1mut-X>rXAe z{90o)mo9fxVy|6;Qzvv|EmOAws7%4}S(8=&vDS6ZjVhc6w0L ze@Bs)E_N0BWikstT$`STSn8v)xST^zdENr{O(<+%J)SPwa=Y#t;(TCEOq_-cHt2w> zMv%HEf}lHkzB-&0O!*Lq;0kFB%y;4YEv8xM^&MhANo#F~?7*c-DBWJooMQPA0PFfH zjhuWk!e19g*O=|7CMpoHE=4~GJ&9|n0PJiC^b#BGa((P8@IN&cu^eCgr@m-=gCn<` z!Y!ez9(pjX3Gj{h>!oIlCEib;k5FGjaMv{5kEp^fH(jT{H==(l6#>3r7fv4cgXyYC zt+z^*o~*w^Iro@R=q(*E0`!6q2%US2*m5FKKuY^{ZM=mhiLW7{5ra*#>9zOy7|SEj zl8_9$*3Av(qd2<8Jm3>y`>$p7xqvNp4nzoF`uo!`XD}ePZVPb7;P;pmT_CC}a$rWS zwxUlE5Nhdj1wqSJv+46;IxA7SqvX`B3$B7l7M&pketH;oxB(zo411|CY6;+9HuaW^ ze$Wf}U{L@BlvbcD<-?C?2e@;KVdt`bVO*{XTBo%qK0(RhcU+r@FUwCW^k8IZOWU$8 zkc!yO{r~XnU<+LgkOlbv&VKkY@A(+Z*%e(N>6I`<`NY8Ywb<5d-?ktvNCDWl8V6V> zzxg%?t5uoyB(1aa%r9<5eT<0!&AqznDJ@V)bel*7ch{*K^JMct%6^^(9Fll}a_CVN zpcf&KHFOk$W^OH#L!;E+U7ei4r~sivDqF8$vIfO^`GAGqpI_-T-O0kCOHv9de#n26&lRLMM*7w82#yxNh|c($ zHEdobU2n8f9U}UJ=XfpBEvdF1o?{ZOJMlTkxQI2#=-;pZExQ#J@3~z7Y8KCFS0IaR z?-a0f*+k{eQC?SUx*mdRLH_HX$ z72HWpZfZnB*z;~DsFCoo!gCQ_^9*s;1zJl^a{4tE;mS=%S7rx%FP&H@1%R|`X)O+| z(@hX{#D|y7I%fIpOtvT_lKp3?%ml4+e-OJ+vn$5~{AA%f0~_6g@P8=8S``M>;wXK3 zL>po?Ji@~7YHLXGXv%o8G{P*`6Q6BcRIPBJW$b(n&k9YXqYDvw%5<47gdJ!!>kR-$ z1UJ1b8>R_0UgPL`@HBKL&wV{iIc-SxS{b!qA!3oV9nSvv(4+P5`K{ zseJxJV5Sp5>)*NphKjy^gLHWAxKyMn>CQc*t9}0-_fws-U=m=8gQQQ=XJaRwMc zO=YCm?Zj+4LzkY)($Q(1bP_^Gtb(D%-yE`_Mj^28JLJN}TBa!MC5IUXLbXbFB;an- z<>xjE3AM|s;;tb=D{JMA!nHW_2&8ar|8wd{{d1MC`@V@3PYk$ci})&CX?A}K=7r52 zo}&qKP>;xVz92t*u2@tuZs7!a-1cOb6VWaN?sf z2_gV{>rXU1>lb60tFBAfzCHFdf!Qj>v*(5zTC@5Q%WQ25lGqpN|4x2{SI*3!L)UJF zYwe?)7JbQ?lR(BMplkyBWaT)Bryb#V79#&-0!F}#plPv;BfgQH=jM#*#vU=#8%1Y& zQ>NFtK+!SONwL^^pwF{7EGoy7tt$|GV}=hK5gWyB-!rl zO$LX;AX3BK%*JPD?1f{iyYq$_5G@eDff+j$x26pe+dX%<@i~1t5XJNbO0M!f%$nss zu<3!5$*Vm%(;4U~0k++46CSv$mR9X-R>_Y_oBFYlR4TC?P5$d-zuqSIyf8C;w?SKl zY|OBC{olGcncIIccn^7`a+{IJh$K#fgqe_6F zqp*;NepbjWeuU<$z;P`8svY=smC^VsW9;QS4}24CFS`!FLK64i4E|SqwffN`q_6)pW+951dgpAN4v%x|bffB-`jOYUBa{h> zU5=SE(yGJ-$cm}Nef&qgz#f1;0PjDJ={@@Ce1Rx$pj)R$+v=KOT4CZ)-U=~f1BiJ4 zvev4Tz9iOgJDJxoj^CTu{*iVY==DEj0$EWl_+USuKtCUDjmlYE?hhmyYp>M{7iN zqeK8}znM{?jx5kVOjMLab-u&TpR@>z+|a*%7+pSHXUa7LILoGuKj1nUEW`HdJ0i_1 z-lMuHq5zEglhBFNE>NYY_@^~3j*D@-q0fP)?7BTlwq$jr-@e~}?@P#m|4QJUx#PsX z?kr%hy5~45h=O4ULaCub>F+rft^*1gS9hAG8fSJIXxrg2;BR;u2Drz27Q;gS0%R`r z5EmlW=zzM9)vNn&+%rBU%?5CWZ*-%&IQw}!g;rJ#rJ*0C&FyGM+L$P&Pecr8Rc?@a zmXDugrq_2NL}LlqS3}RMs5GUSgA8-GX~|P7M%p2#%YK~v7f?Im%hEuWY9q9t3u*Pa2FQ zj0$vmNV~V!V!mun9)RiVVyNT#D3-gx{{2@%{65Vl7OXb_eb}#hNOJt1_zQjSB5DKu zea`$7t^l{z72a*aKjIWlKaGoD#6P#+cs4An^ioe$j5atCD4Zsk5$?|h*0YJeFCrpHM&ovwdC zPEjMvB^TiJzor|5*vh^S9y&jND7Sv0WTU=+5VSb9D;KW|96+7xa?)oTpP&?%<9_tF zXBE0FSG{?NSBPsA5{zZ?j1p;5Fl*16eE|C3$oa&XGDo|(4?T>#gySc#A#p8T?4>(+ zuQVMs)A3LEA;3i;D21abDC6Da#XHN7R+CG|UI9B%pz^pka0;2FnvoUXhHC$@{~x>Kd zUSh3$?lJU0(tRUBjVqh?mINpXxK!uoOQC-irZl8GYq0>?>kS%~KM=Wc zHNVBFWFz$z1ld-$u9n)g15Q0fh-<{rc)P|Tj&EetZNp|3)sc^x+&HU^hjK=5&dOIi z^MZsVM$QBix~+cMFHeg%ciROE8@fHX4v<3s)l4Ok;)c=={grEOoBHnn4-5s# zYrZdbF1z(rz)s(KKbB>?>i>1Y>DUht7<32a7A|uBc+Z6ZtcBhNpRPN}u3rJT)i;<8 z|6QXnPSss_L!x{s31m}#DG(>p5cH#xrQUQ2{xc}pI#_VjagwEYUYHga=!>2mDu(w5 z>jxcg&4Av>iN-t8MBU0Rv$Xh zOO(#_c18swvQ%;N{h%3txL5Icr=c&I>B+8MMxYH7a^eqhGoASA@mNnzm8 z09)Z1SNZA7D=t8$5jB2i5agT>Ta#2-Q8m`10Zy+y!D81f|7^~KS-w0WFs5@fnenye znprtL*84odSoun;LWKsf9E3VEj%8O9EJ#j(i4Q?%WIHwQ5IGUmN_FFcwZ;j|KFhNb1iZBev>jdy`S@DTTFjL+Ly;qQlQ9tZkB*VW>+2ne z52>2ZoRLQb0jI*>@d>K}Jff`_g54DRD0N~hV5<~3NkS#VH2dACZh>O)!|?fRc!8tS zNaE~*yl|7icFnf0L(pvfPE;1vG-UBtwVE33VE&~{$uU!%ATX=$APcR>y!Vp|Pl(Tj zg4^dVphjeRJj=YYyvlm#9o;n}$RZ{d}hfERb?Rwps}3P$ps|4geI6w#ZuTSC`!jxa!Hl+@dn6-!t5 z1(8$=2*6&-R5wz_9mq_)J<|a3Ox#v&3+P8rtL=8bNs@U~=sI#c^@IB9Ui0{}Z9S@`X$KyD?$f(34ZSp?aXoU*f_B5)&&d8(##|sAUn1@U^xfX9w4gx#{;-9^ z7(7YjXxbN>>%QoR@qX&#^d;sF+wu+pxB3A7uF22ac=^_l9=2}y0fG^=jsFle- z<~v+Wu3)5AOTWm6>EST{#8VAF~Q)JNm5fFP2C+xRF<3q(ypzWNj+GT zDc5NfFdFu^jNr^5LR+LjpQ~p{Ew1bTWmMpUkjkXdc7-~6a~fYHgBR<@>UoKHRY9(< zc{2yWPdd2W-N3uCPBB#5tK{|p+HHqGn>xClp}ke?(eWjG@cU`tc47YqpwUa?EMbbXs>LHP-Hba$`I9#2*NBQ@c&!WX!^JS_pGA^E7#%P^jB%Uo{ zGXsP}71OGf_L^`cON+1qv&>d^TQjeEx#hh1QRQ^m`(K_bd41!8;7xJox!nQt_P_~y z=|y<5u?NPe#J73}j3r_~(8{YtSAm0<5%7{10?Bx-8PX?>oahS*GLPVS7UK!9dIG(y z?!$e&jRwdUaF=!~Iqk5~3JM_s7R`RR5hty}CV964Whn<6@Z~tpv_C9W7qGCYf!!fN zJUjpGOz(uo2@)Ao35f zU^d5!Y$eE#{|;0p4DRwYd$Aw*N_;CB4gOv~o z57^q6{q04I;+Ksk1G=lG>^nvUKfjmrKy#3ckPm~hYIMyE;hvQ#r*~>AkA3u_WL=*eSc@xl>unP(N4_{y->anB(_7 zk!)G}qdS|Tx6pG>Ht2qV@uCwBhe zf_TgtqxRoYmjIBV@A~fZA!mL8V{O)v{Vc)Y&7j)9CE!yg7lACKWPJv>qf8pehNgjX z*#B7~xTJ`|R&$JdPPCtz_!+iw2rF{&(H&vb@}p$GzAB7ks7zz$Z+U9~P<%bE5Pt%P zXfCN90iw6O+%I!B=hgE+-~Wh|S@onFz6aHy5rfqt)Y{yqzm8Rp5_;2!+R9B408IeX z;Gb@oAOsPV)QgoZhx)IjNH2g#kF`UR;js8D1We)u11u8Y!hI1zKLiG-he!!O7C!M& zX!*2U(&)$w`|kfB;cXbIOlpk0+W#cNI^l=A!y`np2v;Dp9MR0I_ z6}!Aq;1s0N7y1VYCte$xQWa!LB;z_^J3vS1%h^KbI~jFz4OZQ3#;TWF(GieeZ{g*Z zqiLmjy0CKC903XT8EbAhbfWG!#Ay~?hI=($UmQe?Dv+$aB&zjAIOZ>!-aS@Oc#~C3 zuCxBDR=hJqsHUa>webm`C+>=pcZnev$olA36nE2YX6PJlr){0-n_-|Xi!g^zJP!IVKU@|@c! z%V5@tO+jLepqeG!9ybshY}f1iK$)_dT$ib}N0R{yPR2CF(inonMR(@_OTG`U@BMw= z9YBCp&mX-QCRAL17qrxc?!nD8W_q?5m2ey_=XJkuy z&@x&VtfPJ!D=oy4+tMBE4`VRgDhu$L_hwH(UC?GW(8~Swq)~dxZvPGs;!O9<(FFfJ za&9y}^_@XBdKAQ(kH)`lR?Mv@8uD?Kp?eLmkZo5}1f!Yn}4T^;@vetb9UAl)!2pYK^3Y)YOj)AWvS#~Sjp zCBOxG2wwb%EU`LTm-6EKT7^(RJi6r+0=OWDPL4@rPGbUj1wR$0lXb{TM4vN%L5;b~ zCj=Z0Jgzwi5^Mn6U9*5rNOluo_Yk~Cta$VDoMs~W?Q{AUq0cb|#^2V;*!$fE z4d2}2Ha7L{H1!!vb*CAtUeBnbMOyU$wHxP0@1i7G^|D?;r9X*lH67?;+_&A3tE0C& zJtP?R-txq<$58-Zo`hrQg60LPugzHUGjiX@y9&6X(Kbv(c6I2*_%J(L&qcZFSGc&) zb6Lp)>mCJwDIw28==fhSaScgms{kDK{;}U)#0%?u6RDg{Hd^sdySoqGSx6^q606Tx z!u(SM_X+KNc|&O>jIu#gM`y-;H1X$dWSZ(%*E_>!mG(qFSv5AvWr<9QG5vY<&HZJu zOy6Z$s)8KW%wEtAoV?kv%)QeOkv$Vf#rAAUnAK`w&2Qx2<$VM%Jh;h9GOnHDQI;&a#Tx{-q#_Pq-pua4Hy@Brx0ep2$M7A{5OBpx z21M9#WnJ<}zsY-1a>_@;uY7FhmWQACxlyj1yHP!sQRj97zQ61F!JZaMA=_T|;{BSd zJN0m%){Dynlp#S&oK}*;pn4Cf*}S2FoH||sWwsKClY2}PW@^``%v_opQnv3io$jzm z1*c!waC$t0Srrgz`ue5zxreXqL!ICezyhI%3VoI4JMjcTV%?9=^_f{#ai`gHo+91@ z0(N(}I+zSyFYg4@0zUxKYS~#%*l|z()FZ(A*1vWkU1_0CbR%BAs5qrW8Ui+ej|_y} zeI3{wCH^O+XsygiXyoTcb8a=j=LPU3=IEw}OMRJ=Hs^$1I0a%)t=N?r7QkBblOPuw zWnL;5P~{I+NT0TxsB;d%BAHq=)etj0Z-E_rp0{zdEJ*AE|7Y9tNtaNz0; z-;(`FB`}7?GvDSV?B2K`XKwsY6GV^q!C&5mN(9|xaZi5}97N8LTn%dM%Oj4t3|#u) z`euBjwcN1Y&B)QB|CNyMRL)1cJavu-eGPVuqN7Uw>+;Kh|T*K=8W?6`Twnx(da^nA;KMWM^^P^BZrDeKV#+p>7 z0?6*#p1t)=yvmL>f_Q?tSO2CNfo*-cgCk(o6KnW7fZfC+xehe3^Oq;5U?#`+f}_P0 zn6XCbWV$QNZmZU7aBMr!l%%q4?fmqeh-QrM#E|hAM^NUm#M=OORvH9k7I94>GcBUE zB3d$F!>*TAVG z>WLbcDVg^sSBRN!>HVhH{pRfe%I`HqNsDeWmY9V=&+Kz){f2buoZKo{ySCJ5*W3)s z=Nz@5ovU4+RB9#Pk73VLl$V{aEqb@Eb`L~`>~qqnyK2C^cYm365?X$@J+AgHZr-fU zf+%bBDs8P?d&ev+vg%yRuO!WDfl&$Q7x$>FKksQO`8P~`V2WDc6jZGsI#ANHQvpvc z10sFG1AYKx6L4UDjJ0`|KxJ|QELc7`D(-&T?3x@OD9TO^>&kN!@wXt%DciLpKBm63 z)N^wEbN_A8Zf8nh-%6n5l3VoL~=h@{w=f(yKehj`7GCt9j8M@WEH9zwEDxQPxn@Leqv{ z*@Z7U2RB`lt4ndf$M+!;y6(}<+>P7Q|{VMX6$aCUrmi6*X1!#JRI zV)pgRaUxhZbQFZp*~foq=a(^Xs`9Ye{QKiNwNgni!9x%8L|=S=b7A#Qp%%YBl?nD9 zsHh{y+&cN4Ds__(Utz7fk^+^xi?fv8M#(h$_K~NwpMxRjSd&(Psqi{9^bgBGVYQ7ca99w;7L= zQ!n=)1w73Y^a!3`dozeW5!d7pkJcc5k0=PuLS;0Q3MZPh&DHh#jh^gIQjC>kmKyQn zP;q6`7L-g-*AvD^x7rq?8Wg`I4akl2n4s<9fGAUAJZ$D#t5rsNXQ)H4UJEa?skG5| z>~_y=rElefM+Y+bqcWv>vEsuiMb!>(g>962=C-|1GY%C`O|I%Sl0x}wZw z%;=g1bI+Wt_vhi^w*i`k5(*UW+Wnn?R@<8oRX#PhZ@BS<-z|mz^e+M_CQevkOs4v~ zzB8iZ0V^L-g*c!HyGGt;BWd!{ge2WchIr374kih7B^KlV$o=P;KN?Q(O)ChpToZ)S z{25S~$!XX9l3NRZ0{CvM=y;0+XVkqbqA_3r06-*97i8U2>R0{e>2EM#y%4aIyQ!M} zXj2Qw-tIsFs=lIO802#*Ovnr*JjyTothC7wV(v-M5e#%f#n&1{M*_b=7ET7e_}Ya1ybJ4uXpDMsKJlb!K?c zF+$>G6~P=7fc;YPDy0H834d>IT7WCi0Cfmye`yL>LN0s)S}iN$URV9?UjSRLaMNyj zgh8a8R+cFXNgZy7lBSY5_+H&Ma;@1a+3aRQ(52U2)7u}A=q>$r*}qjDuHB+v{I1dM z+7S26N_Ty385KoKImjXXPS~+uhUZ-2Ot7J@{JQXONRtikfAOlXrRufNbXp~4VY_ZQ zQW!nfPwjHsHGdog#F5~D>;x-}AawKRyQ760yNT9bE{-#n$79>xwC(5kZ(aeOI~|W3 z8t$pLzc<6UTt#PH?STc;? z=;$^_Bug&UI5O=bV_+TIOM&xg5~t7`yAh2uLbuXszp8f?YJmF;`e|U_=&Cvqsl0vT zc9}&&oEZP#G}t{}T=u-%yc7j{Q$yvPDYLu{2M`qqV zjXyssfo19cwTf6O=>J_^IYZs>d3o#8-;Mjk60q4UDd~C|3MCQsX$``Si6#8ZXDM45 zgo`vgY6S>xIUk6-3hud&Zz;WA?lA+6I<)%P{{RQ-1Rian?tuKFa2=H*C{7CHU>SGM z?!e4_NrTee{1AR?c^9yk`3je$YQEu@ME}=G2Ll6LD)5~dy{I#O~-Op3hvX1Bo1qx7DTSga zLd>Daan6Sgn=LtoY&johhUIMLJa(}A{kiWy;r{je%N~#YaJ{$dy5868dYzunXYu30 zaQRrW2l_NiYIkEKgbh1vldTi$%s#ZJf?Yn(R|51y-3O4yzNjC;C*S7(rwhAJ{%WUG zBG#fDzbSBBvEj)J@P&c!OA#T#WHU#P$8}l(%&tV*>mXd|yXdl#E~ZYFHJ|@4ei-K zsEx8X4r^YzE55_xHUyDp#~oaH(Y;Sy_=gWtl77G&m*vb}Ms=%QyuXFNG(vlfx@?IcxyMDb|ro>t(g3tT&$WAR_mR|Xg0 z-C)|^?Q)%EXJldorzvPR#k2cs^8xu$Q{9JK5j4OOEGUW16X;p3J(w9-~=WxJ*GZdZv87f2wS zFAP(3svz9u?%>{0YMKyBukxocv;R(0wrWtb<)R^rG$_s14 z9C!CMK}nOcT666F4fQ}B@7R^Xu~TPE&DP;(ERj8Zcf5EU>)@d;eUGMhqytX}gx^<5 zaKAk`B4=8$Hb_>zB=H2WwvCjxVoROONKUYfzH5=oZdKOpZO^-ox-m++n_c)PN4}1o z-2Zib(P`fnVPw=tD>ae5g1Dn!OaG7hAr0g*BQ|7yNo2~1zo1eSf(Yx6{yP2a#$ykj z{j$I3fV+;1x_i$nlrngyX>}R6YdsMPWlud;9{+ZIHwX^SZqLeVwCrzX#1DT>7kzmg zz9O&h%-$N6N~AM!grEh{83)aAN8ys|RkwW%G8DB~aXybylx=O0Q!V;AzoECjGX*rv zjTM^}Iqm8rGedLF9V-nA401SjPnNzHTr~W_vG(Z~W@xlNlkENWVraQx{Tq|6f+6m_ za@nm6?-&kic<&y0?>b1VNMQf9`v=hL_Ds3uZ7E&yn5O{NWmIQoq~h0#T-HS(D^U>z z?ljISAY6 zXOx7c*AuQ1kKEpxlY^0Jp)5X#>o1a_Q?)?Fr?rmdiw^$mRP?jz{%ln(-$oQVm2?&V zVDhyO0NMR^pyFTlYRO&+x-Cpq#6fHH>oRm$_#9R?^HP{z(%Ebo@Hzi>?nlEqOmV^e zCB6z2agRi)-D7862w!d_WQHt;MY(sEGwTisR5AMH@(U{dDJ$)V@rcI_6uF(?{D#H{I$-bG|Z!gyFJ{tpioxYh#8TtvJ|oi0LHi*kx_t z#FoJ<{x2~?*8dAF#K*E}+C_5lC30i`f({VQXmJh>I-G0>nRe}g)@&BFonF?N(Uk)^ zuxjWYMt_$ggMTTVxe(*olsX~+HYn4s7;6;&;2^nc0$sY=BnJh08K<8N=r}nRZ!v>g zDl=AC^)dCh1X}?JCFrz*IlIF@?nw8ZmVE?>avK7v9=*5T19euuaPRr{TtXE5VE~|k z&M`Z7$a!C|A;7C97IRDDOz7k*PR~p4{({3g&t@)RehFY?RGAapkF}VVcHru{9q-m% zQ|?-FB+zRV4E}8fWVEf-8@2+s+i0ohITIO?t%_w8cT^J+p*(LD3&oK}dc(p2M-OXM zO<>2hD)=LlGE!OM5|kT3m{Xtm4aNLqIQMJNlx%{6-8JzZJ$4eJWoZv%Dm#L71+=%QRG zLnVlAOHi5r4aDmksh&Tf&BV8S_vMxk2oJnHN2$ z4)n~!K$*=5&Unn6VRXl=$L2qix1OWvvsXDMHgoob46UA5=8^zHpD*O5eJD(tBc&Z^|O^+mCe(#k3ei4E8(+}z&iX^-~$jfp*5(EgJ zb_%KD&bpACza{_>IiI&S^Pi6rwwwOJ9uiiicf~_fH;9}JrA%HThn*X(oqTU%5!trz zTV5zRDH-jR=ECLPzVO_M(x1aUxM1eqF(MBqUdjO^o#@}iACfXZMedZvjXlafpvN6G z4--jF0;9n^5e|~iC#aS{xra>n$e0NBA?JJ!UxI6A$U<2T7&_oHSoHWYXCApQCBYL!PhaCWYikZMoD#=^pPl|LlO0WUUudzIl3KjTXP8gap34jazjW7}2z3K3 zK1eLl5Tu5a3|hO1;mua?dCgQt3rW)j|*RNOXNwY7#i=q8iNL`GgJ>9ca7mba7 zxqVd?(YG&=H%A&%C(jnf(!3l0;r{cv`<}9;mPGsoFNcra?<#DfI}um%IJ;(DiUGU( z?zP_6J|n*$`INln7O$|olpH;?Y#8NwyJp$&{jooZ%jGCtT({3?$Q%QI7-H9Vvbw*6uYd4UE7UORArt%-km#Ns( z%!=ul#3n@msc1Cqf}`~ibVYx_d#m(_cO3%s_h83ctnNW-m)iG;2W&T$Ni93nWdI3% zh~h6tcl&-RQ9pdnVOIH%K+@2Iy@iLW3DDo3uT)y*&z~?FnjY7)kh+ddoth_SZLIx> zZ;?s;w@abcr+R4N$!Gy?8U!L%Wcc&&3KhheLcYPn>H*;;}cG~ab~C4wCv3lVL9KF z*Vqsqsh$;gYPK0xbqFdGD)r>KU=u}p(pV9uq1K3pD0RIK&`lqFB&Cm(j)J1)12Bmy zHw%C4kaO^$GaEZbZO!#NmxenO^k0N;+ll4Lf7(H(Nzk`jfYXJRR=#!?*r!g;TsoGk{y*5g(5kJ9|qr*Y3D?k{lspq3y6x6++_c zAWY6wm1J_JxvTd5x#l{ddfI&?16rh{T7utb!nl=pyV*&^iTYh_itNvrS!n69UI5c6 z^_#DZNfa>Rk68IEv%)T`4C5Ca38#%B-jCwh_|xHp?Xo~-@Cu=ihbh_~(77>_JckV|*6!c6G_Q+bF-Q)uo{oUG*HnzdG??x5j)Ye=$ zPlc$rv^cuKDa=hG>zS|(*`V_d>t3?a$X^#&=fyqUUT?SzPDoNh=#a#)@xyBZuqI`Z zHWKf*FRdHP^f*^diR_OVdSJRaZHLSs`U|=_cdJ%3WuwbiyycFa*o^eJUGD@=@t7q$ zQ6%GX$8l0(sE&U$(V^Ig-pAc5Y^n0R5j&Uc_koQlQ~)NgGK6?}<`c*8f#@=cFU@z2@$PO%?|OW;u)=$SrNA)XxY$qE3Xfi_j|JRAu*$0;)6rUWf#^J*Ml6M!AzYfj@C)o zO1%%c8AwJ=z4CcEo^I`pIpwZh{el@a)jG4e#6kY)g+2|_j^}T!q*rdQO>?mqL<+5^ z%;o+m*q+^-Y5CRjX^)QI%vY6R3T*-rzKd09Pai+Nu5?{HJ?lPW-m(A?8+s#-?m*Mf zJ$N|-0-U{VR6b5-v6?o>OgdR=2^Q&QLuoR=qvupT(mcTvqs)PcOhvq!Of+S)|8F?U zZSiV7Bn5+737HB>s?$rm6)G0MuBtVkAdZ7SxQ!qpv=^LW(I>~{`hTCI207s-IX7r7 zk69Vaj-!lS>-SFF%Zv4fVHmQDw!&72x=D-uf1<7nmly32HM1niBBZ=4`~8^pv{mv4CIxz#SW@kn z68hzhF9@$m_LO>#^l01o^>h8{bBcTnm23P;3phT^Ha7FkUU)Sy?G&czl!h_S(VX|e zociz=9S<9>YA}1jUkYS?Os!_l)fqnXlB~LZ>J+k0Hc_?kArJS+A)$)C-xE_&R~pwa zbf~B9t;8TXU#-db#+!|Z^t{~%9<(TI$D-%6gv)8VW0rriPMH3lQ|>doMo+z^v$qo| zF1NFjzc$jryYDy9#N@c)rg6Tst1tw7t(FxB*|ifO;*+NvzvX6t!`WPQ_9!WU#Ee`H zUi+KFR;^0My>dQ9|A>tUw8#M}p!ic;!arJ6BWn?gaF)QAmS{^_$ANxlok zGpaqQo!?~IZzue`BgL0n8gSHA5ZNjM-NRDurgj?xF;QJNNn)z$+Kh3T2|xcDFZZNR@4{^@4|d|86Li%3XF4RpcPt znG>OXovI(ODEx4P57c1{kE)b16B3gwl`lO~(6hew=*VciQ#n>@ zE)UOFV)<3pTRIo@wAZF~NR4jQK!#$DcvDx%D_WGHwq#?TOO;?QHyepB(E-_stLpG% z0=Xa3TG2GTrpEcQ#PFF@`qCAM-QYD6kHd?}e_4IRgSpSpUP5+%e*pKs9^JXy|K8v8 z%NnLPE->ZvYn>}X>8c%9onbbzDh4b!gX@vsE5>&RmG0kLdnUH1U|)AL=B$jGazYbN zZ=u}zk-m2znbTL4M2;PJFsQi>ab4SICKqu*t}s*i+vf@%Ifk{uDrf?AZfo`oOQynQ z)Mz&X&H{+L1J$5D5BvB|FD3fem}^8puLuPrZJDZ&Vlv#bQZw(T%t#(f9j+dl;S65F z9Jeoj?ZXyyJJBH`^ccPSW{>{aw<=Lae*+0SRSPBYy;Nwq48kkdx7eR0D%AK3zG6yz z-z^ffUjCu+@8ED-cm+G_@_`qy_m0w!CF00S813UWjxqqiH5x8EB zo=|h@`hi&2<$wt*N}_RgLc&qWJ{^M2mzM{CmRyEnS#nsvxh8+CkJ$dSb{4wknrgYP zp=Ztc6CEYNFUa)veX#5i$wYNCYju!?TsgM0l*JY4NH^FHAo#xzIDvsByKT|@=iHK! z)B1&(A+ETMriyqPhWC%(fVwT>@&f0yq0HGalOEf6d$v58a1qbixn&@Eb$RHClZw-Y~A*}>bRo}^)uLs|fI zGW34m??2vXE0McSb$)5$K9bEhMeCAt_m8(VJ-J@u-;jG=8k79w!+~c~PEz~SP`_sP z>8|sQsz5*UkG&1#*FNujyZGIMot~@B{T@zri2gyT_IP5Xhu^?3a}V-(=O>Cr@*JdI z&q`QoNXBQ7d9uB+V{L2Y7WHBEbcY74pk1vIMzq(TJ<_44v$uMc@$SNuf`yI(ey@(O z(BdN_*nK3RCif`rOK8azR3~L;WriC9>Fk!EfK#X`gOvQP^~+B^Yb;@I2&5}^ZQ~Wc zFL%Fas&VB@flFn1Uo5U^oiMCZ8GET_wwX*Ba-v$84@f>YB5y5yMNhdrvanWPhDsA; z)U3zblXcCJLAUcj1DoPr<}q+fY&Qix`o_5BqPb|R8;lbWv33=>yLXyqyH3#*EOlUF zo9i@^{7&4m7OS(tRV(3}&X|R?y0=2~qm1#AtkPnD9r@R8t6j2(v?p0O*;{WmFfqe& zGWh%WdIPL3IaVF+0;#6Pdu9CcG)Y*y=xtw=X?E|)&4RqD7Wxu0`p-1W3ZI$&=s*L_VIi0kLxx)RhKlB;^x))F`dzhYTWOaDrV6oz8 zsd5Ry{IlD$3#gHDpI1&Lt0JaHXSx+WV5pHPTC5K53=Vot-)+PQEo+4-51{%`{Mn1n z;ZJajNzdH!@HjSUI?1-lS2BVZq7~!H8|~JTDH&WK*`Ku4rG@wxz)z4F_ouAHAa(+x%vfMIy>u{@8Ae+*)T1ad-9Rvj=AsNevtJ3YnA72ug+c> z9`6~Nr8^`@?eiB~jDf@j{-FKPwTCC#v{nMrm!=gV^Zf=&<`!xz4hxE z@wC819kLJsBGpLg1A~rgf!uLCQYuEXndNNj zFV7la64la6Ps)Lm6TPn-vpmrL#(C03@W+ps!_7#AoUMk6K4Rn}wJNZr)!ePHw$O=h zi`v$)WTUtUn}rAi!vG@mD)Zq)A+8B^5=!2NuPvGq^G-4Yx>?3n?#yVkS#W4E6CK%!o#wdb;+C!G`zSBV{-qPm?l z7pv|M>0WY@@)eMh@U=UJc=h@c-vNQUf)i#a6o$f#=tKD*&Nrs&WC;xT^_BYNmiZUV zJ(-=Wy%o-@3o>X~H*z1zDiv(qT_fy7&9(Zfga!KPuCQG%tTfFrXq;!NIT$@=_!2qG z%{n^ae5FXTe_Lfwz}-l8z-Ripnf+(zm@NkD@Zf}hOn32m_=eo$@PX+6w#0z>UFx@D zbEJC7yuAIFEQ1PACK^*w$|qX($6PG8TVu(!bhX|J=PEz%-!ub;{OM-j-*r!46_P!Y z?6ObbQda198O{D`aoLjR)dtFs?rrdp&%t$GajCue+x`C6i=^cxGBB;>!A!VIJ=Mz! zg@sohq(tPD>{ttRkioCn5)8s27{I zvAiA3bXpE~U^+~MH1|qlQ$*ZGoz8I7T(hkee?LZt(QJND@pbcWfwx(b_AK z3hd#qzGVf@Y}ub(g3#waYUU=!Xop`U^~kk=Un-6BxuG4p6JmZb#%hOP4^T?O2b=&% z85r8V7#K}t2H>23mI)majp5E*QJMHQr*rM%FJfkOYtXwYsL*rg! zV!MOK6pRX|UPlpICYfdKSq0{VnV3M;x8xw0v3FYpI_I zTg?h78|n|j?!Ert{dv&kaua7pH4#JL()R^iKIr1|@bQa<{6HX~-e&VrQJ^@QJ)go) zF&-Toj~(?8jHOj#eTL?ZYagHesq=1S$XMl>WJKn|^wy(s%n0nM1nu&5sIxkLdNHjd zUS+3kq^tY|Ex6_^r+nMWso_Id{JLvy@#Xz%q0_~#=2XOV<@HjHypQn7e==joK^zNm zv&9!O3VyAmBdrByx+A!2c-R^?go0 zewq%+o7A-qw5@vxW2T=-p_D7X5d0L+z=mXjgz&0~{UKQ=P6f}Mj+BMsr}e7X+Xh#n zwpTah;s4U3))v#VZ5@d%e&}HwMLRcg<&$1_w1;=j0_n*~Q}z_KQO9C*H)q;zm7T+B z(PHpdBmhlP)Y#$gox|%rCFL8UqOR6zPDiT6!oHqvM|OA}nyX$Um*SvjBC-aeBY%N`)!L#lwS`-OXr+}wjxo(H8wttOn-ojtcR-uk3)G*M$b=&=SJ!KuZk->B5}7zxJ!#wUMF|5fv}`gKH+))L47eoM?m1 z)6Bh4wtM{g;`7fnw~XxDD3Q`c74vEz=nY(|J7c5recu{6@~l61d~1Wzk+g*+h*|Cy z*Gy+`L7&!K(*)w~ao48FZB~T*yjR6(+aPxTTjQGA%-(4^m&Tm4de&)K@2;ile>Jx`Qmd zHKp_hoib%Sszc!?W$9Wb6lSk|4&7Gb6dc1nC?4C%hEb%kp8M*B1}Sb*q`dHPzg!fM zNmq_PtNUhiyYa0)eOCO5`8v5qPd(3Q{&rwmq2qen}(7DQsHW|eJ93VpK7 z9di~=89_H`i?(%}o+~?o7u!{$#Se$$tgSRwG_fEh0?>l@E1xG>*j169{i+4PS&5*^ zZfe^ol&Ph+BWFxO`Nwdcr0g(vEpEic6OeB6QZG2sBP5~pXd|lnp47frpY?Lb!d9YV zVLQKSc<-&~iGfG#6&T#*Oazn84v7TC+l-YrV#mioW?F-XcJzPW`%AilHDaDuipdMX zSjc}Gd>;FCyzZ71m^MhnPL|fMmq{5NEwAU$180yE`bW?-$-jexlXefbfMEU+$4z(& zCL6FJlieM6{qyflVOjtE*8)cJFC|$CP8^{{JuBW?$$Umk7nlk><#ePOLq)JT;Uz%^ z&q-w;-a>Q(ohdzIIUA6S(&I>Ujt7(AVGo zM19A`Z_)9&jp)?a^|$ALm-{`*R*EL?Ci)uwQFaT|j;R-FG~A!9RCg;eIFVUtU&ri1 zsV3B9KYKCow-GjRO{ke5Mr936!!+bxV|_BACorDf7u#|($aU8>N3BeD^|9O%R?ykv zF=P~!1y%Vmb&>CY=({ovl~AR+zrjyJQ%*$nKQsNN#pV2YGyTLk4-BS0zuf?r9EZ_$ zAo}>-!r!0*kn`QqN61U#+q9tNpmtt&o8Py;>#9)RG!^hY(?I zr+qzLej&q|LXCWjdtJIXo|=Y42ZG>K#81)oLVf0*7J^OqaI%aFhk|zBStf36g|nB| zgTs;A&X9)aFzrbV+A&c1S<5bX%a=9$4@2OcKXd)cu>zBT5mCuuK3Yij(*P@+=jlwIOT$2PRV+Y(M&aD%Sjv?4xSZVXpAK3w zX0I8vqY_2BUw1kRCiyS&TduB5-3{iGu=*3be#7}sqDu5R8l%C<`56I94rfFoCiqDS z!;jae-9m7T*TGacd)hpWpgzF`S{Wq9?*7Um{O<8GgYwkcLl1wqtNo|rvXW}Z@ow4$S z&-{yEeuLm8PmgcR8#N}3w7gQBhw@Nfz$xw*>UHUgvGN<8El|0DSdCfm4FlM%dP0e_ zN%57au)PjtN92A67z&4EZe`wxYK{dqJ8%%NEY3j7EA=tTQP5#LlK4k2l z-tRUWwT(mmRp^_$*8QK$x@j}W#+(cdFIdg(H2NRfV$caN4WA@ye~j3eXGz(0TSGFi z*Au9FubZp4u#9(}Anxc|onCmnaY;-iSp|I4+aZq~7pNa-qakB+!}`q=B`$OAwH)Vb zs^{cepauDY&sE~_Cb5M(s?pObvW52}E;?|nQlM~dc0F7&q`!0sLkY1Rzk$UynvlT{ zW2@R}K8(jh>$rM%^5E3LBll8ZxRMPMY7o%FITZs02c>&nSg!(f5 zTSNDh+vl?XM9!oox7kfVPKk6~8DAk|pnK0WiJ@w

UfihaCQirj4A9t(VyJ?^5ek z4ZHN@8L}%#vYujuh8w7Z)!H?E-H0tAx&@l?KU*v}MoVK${0ZgI8*l#Z8k)t%nvlPj zFt#*tcL!mw`xw8(c}ovx!BhD*^SOPx)={xPmOn=gTEk!E0qv`@>Q?GIW4;x7^6iJ9 zXl~ZcGxLV8b2A>jAx*6x)9&yUGy=*1CvIOL4}bJ$`z)8N5gdQ4tmukXo4#u7CeSl} zj4HEF3xP^Fvl6tsgUp{=y!^R?0*v+Ui4%;7IV=qirAV1BvCv?u>vg56G>QLy7J!t< zf~~%qD))NlDrp#LhJ*-KI<|$36UPtz@jskbdL2hNT+v1DAeSB)sA?<$$t0di$fbuB zvCwG>d62O?5o5&UX%>^9js{%0fO%j7)!h@)a*YcuZ9ztGY5$lVA-59zkKTWQB{#Xk zS9tcZsl%qBo0!+|Y|2`Pr(AQg8T1VLfL*qi4o%6s~K)I79= z>KyYmPY>71F_5+>EbaX$-QFQsxP$%6eiF_&I^61wwK$?d{h&EK?5-Q*1AIqoiFCitB0nI#V#VzjtzF}mvg+eD~`jM8A@XrKms_g z6Sdyq3#$a?F3ff^LtDpCJ#_rjiP;@bdsia%_(_Qy&f8I1EA_Gbv1!Z;0!d?8s<_pk zVc42rU?Ee0h1{yWD|d!lecQhB^j{!oV;H_XJywME@#}A`bR_r5s)>}l4~C4NbW|#< z9Z=DGOy&0KaNp-7pr|3~A#<@`m={preV`>QAK0OU5Ws(P|1Qp$(_@4Irx(MI^z1|H zs;a#Ac;+7yGW&~+%oDD5L_LqB7lB`Z-RsAVV2=GnmaOTGyRbo8e+W19@%oZf2hmYM zz$k%v!T97HQog~2x(^*y$2W4`%6-bznU!P=*06sxB=5XZfSmWjxAuH@*e*cQVU2ds zBJUUoHa{?zn^7U7JFY<(Un%rvkRjwvXY=r|5hwgZTnJ0RP8+Rd@V0ym zi5RI{`fT6E1lRltHwoBWN0h#fQ>`34{NqRJTr2z97ll8iS=w?Wa;;kgb~JM%WNv#d zW$AFCO!jb$eSW-gO?78wBb*-bvv`f=99cJsZ3{IxgFNUwX$SECb-ceRsg@ObD!G0% z<}7x|HW-Te_R$1V+Bd6uIt^rMi!Lxj;$wgC7O z?`lZ;4#0Tr|HhjrOkl=?Odp5S>&DbX<4d^>P{(ZH!AWCb+M9TFxYY0^(a4>^wZ610 zIaTax#8S)|T3`6J=t-o1V<&sg@qik#%r|mHMTGumBQ8ML zRN-!TH_=g-b`)Cw)P_!x8R%fI&FA*W6aiId>r3G5#;SKE_mO3fYc$~uBMx!@>O0+# zRvFK1HPp2x>)?0VjE)O)t>TvNZxFQ%>!gYjfKf{*XYL|SPbcf8?(Rgyl|xWumR(`h z8_&$Z#>L|Tf4QMMr-uo0q*cVh5+L``>e@bbQ=_+&`Av7~jrjEM8>Ht+J-0Rxfm+TcR6w#L@ zzig~9y%7Jc!N4XBFLM4;1jvC;tH+ z*78+%*U8zL5xIg-IJ9uF5fD=!0<3>dq-ENa9gQpa4?tiY`Zic$!W-nKbIy{A?Os5U z<*9rBtSA4qmRT&hM}aJ?qaozDWIvZ?w%Aty@5sl;2euOeK8t{IH^6fU5QG9>N&tE@ zKEC_>`=kNBM8*Hu8TqUM*O`y+>yiIAui_4f)@S{qKeO-+B3;x%i*8{NH8q hzq|7P+g0Gn?I{RpTIF)D04@ig@m-5M<+oj;{txg)_F@14 literal 0 HcmV?d00001 diff --git a/.well-known/ai-plugin.json b/.well-known/ai-plugin.json new file mode 100644 index 000000000..bc08de0d4 --- /dev/null +++ b/.well-known/ai-plugin.json @@ -0,0 +1,18 @@ +{ + "schema_version": "v1", + "name_for_model": "text processing tools", + "name_for_human": "MetaGPT Text Plugin", + "description_for_model": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "description_for_human": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "auth": { + "type": "none", + }, + "api": { + "type": "openapi", + "url": "https://localhost:8080/.well-known/openapi.yaml", + "has_user_authentication": false + }, + "logo_url": "https://localhost:8080/.well-known/MetaGPT-logo.png", + "contact_email": "hello@contact.com", + "legal_info_url": "http://localhost:8080/legal-info" +} \ No newline at end of file diff --git a/spec/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml similarity index 96% rename from spec/metagpt_oas3_api.yaml rename to .well-known/metagpt_oas3_api.yaml index 70c15d590..e6cf25d86 100644 --- a/spec/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -56,8 +56,9 @@ paths: schema: type: object properties: - result: + wav_data: type: string + format: base64 '400': description: "Bad Request" '500': @@ -96,6 +97,7 @@ paths: properties: image_data: type: string + format: base64 '400': description: "Bad Request" '500': diff --git a/spec/openapi.yaml b/.well-known/openapi.yaml similarity index 100% rename from spec/openapi.yaml rename to .well-known/openapi.yaml diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 5d0001b27..6b1a041f3 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -12,7 +12,7 @@ import base64 import sys sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initalize_enviroment +from metagpt.utils.common import initialize_environment from metagpt.logs import logger from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py index 686fba34b..e1bad6456 100644 --- a/metagpt/tools/hello.py +++ b/metagpt/tools/hello.py @@ -22,6 +22,6 @@ def post_greeting(name: str) -> str: if __name__ == "__main__": - app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("openapi.yaml", arguments={"title": "Hello World Example"}) app.run(port=8080) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 921629d8c..ef3347b6c 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -10,11 +10,11 @@ from pathlib import Path import sys import connexion sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initalize_enviroment +from metagpt.utils.common import initialize_environment if __name__ == "__main__": - initalize_enviroment() + initialize_environment() - app = connexion.AioHttpApp(__name__, specification_dir='../../spec/') + app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.run(port=8080) diff --git a/metagpt/tools/openai_text_2_image.py b/metagpt/tools/openai_text_2_image.py index 3d2a2bbfc..50c007626 100644 --- a/metagpt/tools/openai_text_2_image.py +++ b/metagpt/tools/openai_text_2_image.py @@ -16,7 +16,7 @@ import requests from pydantic import BaseModel sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initalize_enviroment +from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -94,7 +94,7 @@ def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key= if __name__ == "__main__": - initalize_enviroment() + initialize_environment() v = oas3_openai_text_2_image("Panda emoji") print(v) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index b15c1d186..ea6af7e7c 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -260,10 +260,10 @@ def parse_recipient(text): return recipient.group(1) if recipient else "" -def initalize_enviroment(): +def initialize_environment(): """Load `config/config.yaml` to `os.environ`""" yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" with open(str(yaml_file_path), "r") as yaml_file: data = yaml.safe_load(yaml_file) for k, v in data.items(): - os.environ[k] = str(v) \ No newline at end of file + os.environ[k] = str(v) From 18ea97fcc60dbdc6de01aff374307735d424a050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 11:12:35 +0800 Subject: [PATCH 042/378] feat: update ai-plugin.json --- .well-known/MetaGPT-logo.png | Bin 50622 -> 0 bytes .well-known/ai-plugin.json | 14 +++---- metagpt/tools/openai_text_2_embedding.py | 47 +++++++++++++++++++++++ 3 files changed, 54 insertions(+), 7 deletions(-) delete mode 100644 .well-known/MetaGPT-logo.png create mode 100644 metagpt/tools/openai_text_2_embedding.py diff --git a/.well-known/MetaGPT-logo.png b/.well-known/MetaGPT-logo.png deleted file mode 100644 index 159517fcd4f62049f43eec4db62e1770d189ae89..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50622 zcmeEt^;eVs`~Of0QNkdUvJh!SNeKl(r9@%W2x(!A(MSyy0Ra&N0g)0VH^#_~+@!l> zz!=?QG$Y2~v)A{}_@4I<_qp%gXXk9s>v_fFdR&iZq=Ei(HrDH`004mP<%?%;0D!aq z{<~P1=x>fjI8*7j^X@OqyZ`_Ro&PR|9F?R3`b!3{H_z1pMLqm0^bhBpp6WdX07_yn z9NRDg0L_LkpFK5t&#*>h{%rvqrfm5v{<#tNDJGsN$t0VJ?}qSOD9dZ5cbA_Sm|u>1 zbo0i5&gB5ZFE_qK-FaKj%5Y91^TX9(NSs#56V`jnsbf=XUcP43F3tAr?zElLnT5&6 z3tcIZ^OH^^<6x&;P5RFR0Qr^S|L<;a_T1b59-c^Yeqi|Tq2dYPzm))n&M@%)cQ?53 zzc2jH4*&Cq|HZ}s@&i2&{4Y=buN?fZ9Q>~w{I4ARe^m~OqW}Qoe6upg_PJQKjF7HO zHMP@XYtWI*cZt)btyU*4YbUdlG3&|d-+MUKvA0c3wz?#OR_0gU2Y>c@9PfsXT$cc| zS?Pqc35lVsz;PFa9#}B33635qXZxgPo%y}FeH!?~eJrH(ZmuQ41ca(hJRR;nvS?Tk_Po74Tuw6Pm&YFgj; zuVE*HcDfk>#jck=FuWBV?TDi-9DRGd-WF0q!C8=^`s0fg@jrQCWt`q+x2EB5U2zx; znFJC(O=Gs4+Zb#^LCeoTVYMSZhYKeGfz%ON_TH7>-;uXe?-4Rt2#{0#Yf3=^WA;0k zw%pV@v&lN*3xv|i ztYe!7yz)Lt12x=+sMq90V?@>6E`UO=DPnfrhr$_LY-hthaTvMaTumh0mPazheH+neG6e~4Z!=EzDBRR>#7 zcC0m^z=3E6MUGyn(M1MDari&~nsUT-s@!V$g<2b;``=!bB3y_@x`t~#1v0oX343_> z@A=diYh!AgR;tVpb7xN!-R7*W4x`~V);E~|&*QJIkVh%Td9&H3id*&A5 zxokY@2TcMY2jntx$7S(Ge?-~$OKOZgJ{X-Wi5WBaAS~;(n0^=%{V=yTuGLB9?lF8okst^O-5f2kGgG?= zPzNv>*s1PXd3DQ*JHPnGT49#UWLi+r>vqOlP?>gp(y?UW^=&BBm;}?~2BnpavI^v? zMKG9ZF0+M}9nb9^!rfmVcH^%S`oP!Mk0+F7x_}YG0W-&A`C^`p={QUA+F>*_(Ib=k zq1NXL+PXm}RdJ_Cc~ie;(L4qB7DhTHAW#2dP9PzAqDd@|7C5iP=Vpjbd3!2;_?TA@k00R^|#jNeaXFJVmE8KA>AbnBv9%O#R)elua*Ih=|R-S za0bj7En^3&ZE@_QNO52CaE$R2Hz~QS1*HtH+y3lXw)`VEc_8>kty+C{HAj&#PXeYb z&IZN(_T-q+bA~aw{L=;!c`7JWj!PWP2HT-Vjn5`*6<7nKx%~Hl}6-&WbChIcX6EX zo1e>t`;6#7W_b%j*~`Wvj;=pq_1L2{^edfF^Y(+XPTS3%XrGkKXO@#Sl{bYBmS6a9 zJeN!E55(#isQL9Ar=%StpyVqQMls7GOht;Mm2>@Bw}s za5i7Unw&1XhHHfC9YA`O*QIOYaMoDGcT{8E z$#Niz-YlN!a91kOg zv-aH_THKH38n;Kiz3o9PX03g&M!E8->_AaBkv*Z1uY^pf0LmNfjapG6>o7kwzQHmJ zpK})dsBxSW$HVrt4p+DooH7`9T;zv)&3H%GT^`uz`9&jCz*Wz{MN<|VQUUdT30Kv$?+E}pRJ^H1!alLz?E{dqC# zv-}a}4mR%+!C}5OW>7C9-H+>K)azVMiG^HAw?^eAZb&9H14w78ISADbO2)7`yZK(l zYki6O3Q74?}a0nB6mvEAp0q;=&3t3K9$``TrY;>4?&!KBjb z-iha|h=NE$W(-`AWMiiOcsELmlc|ry@(&Q8FMZBhhq*1AQDG>=r=}RIk71;$z=Ao4 z3H(*$=wlFjAd;t6v zCt-}68-F`|E1m16LZPfL=E3^tp`@ilK-~=TM$o+P_Lhnqmv?N<#<8aOfz?lpvJMNj zh4xphVj*d9yW9ywkzYSdWttroBY%Vb%7M!yPIE=xxcJ+I4hwd_TAjy+@xJc_PNIJ! z&SasuN#vL&@(k$nsO^(P0i#4zR)u7juW2Iu>eBF(ymO>-Z!I3(U9&DnLct%1|0JR>}toY?|QPc0W9+D-Y9JZCQ zeaQ?u`lP>(qd zXcRiQ(rp7lUZrjxo$o$uatx{CsIp-#cZ+drqF41^tuPO3Ms~s#2bbT8d>J|TmFj8B zLJcX5G{WmOqz{Ih4qsLY8wM0BxADZzh-hH+z9pF|Vv-5kn@^v+Ns=yqHpw3=%ZQu% zrmBN|IM>?DSY%nnQYDr4%lm!euE}ts0@-YX(N(+N`1sNCz|q^_8e2@UckkOmQ{5P* zK5@FER2%!P&rzq!Up;n%dL>m-j7^?51|M16`$|3F22R+1Dr5V1*C?5){-na1s_pvP znH)WAACmH8alS!gaW65krLrz7@OJL1SrE-y8>w%H=c{ada-4CYF74%A<3$e}i z-Tuug-UP`^l~tnW^mowtBoyNgPrCz8u%1-ivXWWxTN~&5z_h-J-J3&KL(28aMk{3V zz`lBlcWP+C(bQ}%t2y|;=Gz4~){1l*WCycM#0WWdq4xpa;%DxED6&V?p;y5OT-_}r zalJsmF6o_JUrb|>dLWAZJ6gC(n!Q}C;m?I~^qX;S$J&bB54;p<0URu zMS)?Kpz`s_dpm}|_^>i-Bm4;Ta`?D!`!XC&cf}5Omhu(8Gbg_OR_6^1?fyx+B^21L zxJzD+aFeyKvk9I5cN_g9mRyqcc8offe)@`C(ftkL=&|(qA~Tlacvq((*J^T9d zoFJhG>-jZP>M4x>0k3d{qu~!eMrbERiCM6&JCMrEtN_*s^G!V1Z z!v@_Fz-OJn0Zyizp@21b4OX&hX5^HcWFP&O55I6kS(Y%0%P9|jqha!S*!MYGg_da( zsyAk;^=(?%2uVUQM4m;5ID+H8`J(3jMBC836Yc0g+nr^pqyuEssQ199I;8+#oqs+& z%9~UgOQwt%%O0ect`{?W%8_>(x#HROG!K1d6J?a@dHsb~(Kbk;UlQW*qg))}0wp-) zs`P0W%0x&eZhsJna1iF2KMwEatt@%d&&~Fmj~>_FY=m`J1XJDM`dagopUg4|RmS#H zGHF%hPrE-Bb>o9SYJl9W3%eY#s{|HE=jDb%!dV4zv2scP^=EJlaOh}46=?RYv3UDF z5pQx48}Audic9}2+cq=7R}!OC$i_~%CA$PrHl;`Xmm%~ zK1ud)S^{P3t>}f>ni|(;s;qx_iuzHfT2>Xb_yu!8$?s58zGE!UTKK}E^p6wsq|W+b zRC5f*nG%?)9!M&~S(ZIIKQ1JWGjWbD7ph)Bbe+^jy`2~fn63|HUJ{ZkKb&5a-qHHX z+wY5`_iw^e1o@q{DTkxK$x7x)T-0j4pZ7na-vKN`0T ztSEc@;7<4?+Q!t4wVVSp6BwB|VY&kOOqfqx)#5gZWFU+VTn2KoIRuhLP+_*ZX7{rKi%yUbWjWXWaFhIy}9DoQ*;p9OW*Za%v+biY|Oh zG{01bgO;+rsfrm}-UP?PJ%7p(AI0l_mIB6|8nU2geCOM~A1-yvPbmi>L)W+8o?>@q z4(geOelQEZzj0bEe|i#lxCRcSk}-yCOvM9qM<_I^1MThE@^)$4$c)Gi-zXd~DBLEO z-QTN{E*FUwG8i@$wlF@|fVJiO%UMlTcNpfV+ol&88e})@#@M_3$X>T%W9&E6p#aM% zTO^aWaPOEv8QDLo$d$B*^Pm-FoJoJU0v>9!%^9^ui8lqPduss4i0W^)J|2897C!iJ zTZ{Firc?SPH!bjJn2vyXwC{DN4wz5adzPJYHS7|OER4ME7)^w3pD(w~4$ho*d^0>8 z$dK1qI&^LYre0RpOW*SC4`4N|nCpp9Vp(mdn}{nfdS=w^&=FT;d(ZI&_lLxo|F(IV z3`gaGBFfMkgOoYmiEM8(3+ahui$xJNC)kD+&$k^h#4>}4=5I178g#4q{@4=y4-99m zW;V{;k3BeuQRT9*tEiX}Y{cj+J*`5$0$TF9Si1M43qg6o2*1lGorZsiCC5tB;nwnn z=FL7pA~bJ4_I2$@GdX(XU4n8;syr+vZEr!KfC=|mu`DyGkwe)K=8H!~cZugD+54n; zpn??3!+*o!m50A9eFzd3R`A7DxZBBT*Z`+QopAgkjl{8K(Y};%>B>F-gD*3sol=k6 zdme`M{YLB zBA7+u(Ev&NrsDYqHvyEOKCt{L>N31808-3R9_&Buny2AH;>xjcSca9D8rs z@3z*XQaIx?OuWz183we@Vark!uXt61wl}dv%R5)P^NS6d^-B?5TkvJP-ojbkiV2k& z<`_jP9D1ouGd;N`8I)@Ad8<8baekJ^tbv2~{0J=XJL$f9Zz&X;<0 zbpLwzt7}5xG@m1M_b&RDO-ymoRSe@Y3{mB5x{hKcBd`Q04(W<(M9VmK<(bd8e3sCm zY>3zN;4u;;S^6SSC)KsSqs*#PE=aX~Oi?AytN7;omao?im#nM|T{AJn*>5lV0mhH$ z#fhusWZSnxG;bcayNlM{MXOb_{$e7IGPxRcdiV=KxZ#y-CT11&zFa&Ob~@$fCHs+} zRBX`W>@MOfi~$L~e7|lC7~QBCwpXiJ+!f(CR!neTa)=_V$~LM7$?r(a!b=Znvqw(t z-eB(%sVdV(1pY}Ll=TjI2#YiWhBf}a|NYiQDv5~5xAuClwPmsIR+401_q3#yfv*F0 zMLtoM8T)3{b>`F3!o(m>%*4vG>@y@xo!~WCm&qBS{qkMJNZ=_2b0WtyHShU4_H}iv z)dm8JpIls1chBceHIBP}FGBFd?Ak-$jbyG{r&omBA;r6yd-o9w-6##T5L%CbdIb}e zJ4cYe8mM@Y%3xM!p2gf3NXHz;EY!|1IuB&}T{Pb#ADFg2(R+Q0(@pcpn%SagjS_Z0 zEuvp}=n$}$D^;`7Tk>WN<&}k*X+&}qvGot;(H3mTP(fwt(2as{khH{Q{9TsV<-ZZL2vwv}x|gnKEdk+rhMynF7}BFPiD11(2dOw}a{_dm2S-npR_F_ynuNlgN)1m$}sB7XkQt+P9;-5GR_RfXK zfo}BNSnS8p5m#WK+TfAGJ=C$3DAg|xB80BT7d<5p;}TpehQe)5v(A+vV@tra1!t3=(pxUQ^^rT z;2p!K#zW7RRpl9#ismHYO)0NIypg}g*YIm-?7DdHhDPC(cxAsqZ)X$Q*;q`% z`l+Iu_|O-U4J29RO732v|t!Mw@xT?LehSDs&7cc`baqN(ol z@aN^tysHhRYJRVmW~bufHl3F8G;#ia>phQvpiU(XlVmaT%4wm}Q>(nrU_Uhr)-*D3 zB%965@-!aOlaplvbMbz7&3)Jv!A}YfX`aHC@AabzI30Zjk58`hE?~1a51l53ALdSp zS*QD2k~aQ2?Rb49PaaLcTf=8c>{=IXM#JZYz>=&#Ufk*HmM{f#9o*$mx`~+$6QqDu5m}9u) zdTRKrgt9bHRtgwc_U5;ALk^0cWK7c9K02g$)CtP3Kd`7(onbqN$%F<4efJi6i(`6x zMdf|ryFA}|v)S@@9+y`>GpAG`p%+QNO#H}OKfm*XkW_jLKCjNi!M_8S}d5Px0K z!s>tGS{09%Xq|47ZdJnOww!K4Zo{fC#C5~;8$7mtQiEK)Npn|_(=~|_1t7;D6REU- z$gbKQsV&fh3i4yu*i{PoH`~5)a)e96E}SWsvv7rW>T@q{Z@W#CRBA z$@n$%WZ>H&1|>rRZ*ycfqDeCP8AgVOjvOu_{SPvYB=7E*m@9E z@%@p6j;qf%sLVlMT9w4&XGxmCTWS(_35U2VmaT1-1y$n-gU8e1cjc$T6no_W9@ z>f-9bUO(h%BbQL=i|5E4F#Pk7Cn@16FvTszxYZcaa}$s(@-&qs1B8H&FB0Kbm&BUL;wWIkIe^ISlN(<1RVb=PJl{+&DV_dacK`5&aKg*;VoODgCqI^+H(rzg)`UL(~n{Q#M?#+T@8 z>5Z|oqZ7~H5;Y)#amva8l(5XCC3PJL@frY1z3e7V#YCK(Uav*z;Vwi_#`Nel{RhOns z<<9ivt$YLHdV8x5um>f}8tzd2FBzVT7?9A^sz`or79~flA!`7k4yk%$QTtQ1&D`aa zx1#QTjpZ9d^w>HXIZdrH-VDx@IL{X(U9~Az<-W?LiHU^bjTtccn^Npt%L<9r{7rLS z&-ZVgJ8No8M;A%v!*^A9WVl>uad`@TqUc9?B+CRqnh`hgI-yXyTrWbbm<{Q+YPgX&-!4;;v_r84VFZmTo-bqnS=h`eZ18Bkf}k} z)e$puRwdUZ%7V+O0--h-k^ip!$_kv3h3kun$l{N}csUQuebgp@&R>(?0|>Shx9H-QIE z@AQ%D(j{-wRh?AU#1p)yO_Y}5zn=dJK-A zY7+!M`nA1Tl$$L%w(RK3R%Xd_5s-chaCZ3%S*$2TLg!TsJ1M>qS9XUt!hpEFsK?ZPh{tJ+@K;88|JA&3ax;+Z zous^-YBt*l83^YWL*FX7E`IB!Y-OfTU5-0K!_GseA{m-X$gylQmh2_C%D<7SoxO8S z)z;GNa+y0{oLgD@V-B(kp*x zuvS3vo&RFUbhr^!G`s4O$8PORDt1Xodc;XJurLZSIlsRJ{zyY>A$IHj+(SnadfJ#5 z9l6b8VsNi=bJY>S?}JHR4}NPy!1MT4Iotc+uH?=ISP9*oo2)krx^iO&oUB=)q4OR# zbD{;^%g(^L_(^RfN(9h~I1tq?X==LueWqKZG2zyognL*iB-;o*@72vW_$q5`Jur<& z4R-ACO({L;!Of5@c|Ji7h=F-t^ZgNy(J!d~5bA{;tJ-e7<_>q*y2M8Bk_TT315wD~ zR{gWP#g|BJVaY<$1lLnJvp2~WkMw$u2Q;_W9_}H5qdF{&f9VR6rPd40BNxTk ztgIa1tipots(6701izyi*S~9-Ia=Uiut%T554JtZB z|72UMpg|IQOAz#DF53a;R&^8BOVlEO)6fd7b~T;H)z;2mpw9ZBLCv~LGpwmG;k}P$ zN{(sAkW{_f0noR9zg;;!jjPHsVDq2Tk~@b(g4H+wgq_9(`zoQhU`zSGMgE9qRMV<{ z!?SZ(QS2XJNOud3!jTIiiNoc4qf>Y**!0bsj{eqI(5&y$e@t?ynpZ7-EF#^Yn1k#L zKtdlXVt8F?+dPBSh!q?WVFOs52A$G+SNK?%HLhX3`izevs)ds}tr(>`DePe;j*rEwI zYmrb>j*y7qFi4hPhuLbbh$O!pF57gur@!g>#n#}t79{8Ko}_&@gSh-TLdOPHNX=T% zIc(}rx;z9(2|PeEPd{2~VMWy#F;5(U zdPkN+kw(f|hUs2M$`(M35mFz&pS%BZ9NqjyD4#1-u3F$G@4t9@dU z&3s?TZSbQ>$R6IbCebh zAn?Qvx@#b}_ZmH7yiYty_>lFg-p$$z+j;pk4}wFfR3Puy9|-X@KZwADb@ z3!*$MKm@Bjz0lmNfopJrYv~xD{kFq%v?}g(($E4A@rF}|uRj)tTCO*TkMpA+XiMoZ zTO=~uC5ouDF7Y*%1$Rf`pFvW7G|Myf@wu{KPfw_ffRog4z`qb1j1t^;!Q{?ZxQGt! zTjrekph2@YSCfH5w+!$mkjqF-L9CDpAD=Q%|$s%MO^Hy_e{RS~SVS8?FcMIX>@SUB~Juv0UHk z9Z>lei85MI{F^+Q`I_Ry1vr)`R%M?3Q1pP#8bqz{AS&IBingO%5R$N$NxTQyS4U(7 z;(eS(lM-Rw2yaz}Ti)AmP}6fb}BRgnJ9X>4=>%5omAS9M^e4(Q5!^_L~O zTFjzcMZO`f;Fb|$#&lgSo*Ai-I9A#BGcy_NrypALdGi-;H*3$r=msTmRI4)hJ(ADt z@~0ipAz=8+Q^Jb?)vW5&0_K5c>rL`)QSL|Q2NxFk zU}k9_19x)r9jiBXDtb#iuZvg4)c!F$zV_?jtIYWjY)hSkVRrnbY=xe5Yu|r~T`Hn36o+E(lnG{v{1N#OA^EFGC26iGTILkDsTHGUGT1dl z4sN1jzW3yeN~(0j=hFtw^yt&mCmu}bI$zozzE?bkLi&&Nw^#&0$#|Oo@m6c2Q%#!v zNTJ!G6W{onLt;p%2V;pEVlA_IzCD2vV8trg2e29t%}cu_k{MKLWPP_9sbCyzm+GE$ zuEm=M&2I>!c}A~v{ndgHkA(dblsDaKXh|})T;9ndHGufC`Gm>nw(+El$rFWoq}w7s&` zXzyQruqM$+1Nbo|$< zpuCZ=9k3^YuuXIQvISh0sG4vZ4ZmKvu(9FkCAb{DYV*LplXwf9$wWTuudqL|pGj?C zk_nhlDpelc4BS#w^Grw?l&w>`5q>|7M`3UJ-uLZn$B-(H8k3C}sy zf?upM@>7r!C*s}Xn9$pYX4D0^s)2|hnxFyoOcy2g603tw zXQL#Z8NF0(Me)s4V!2F*=BiUy$qB~~WWL)UBsIZHQ7k;6mCt(WV0L4!oetEN)>qim ze11xS^C3R>s<$3Xkr*$xvsIXKun-OdbxzHG($k5ULZFhPAxYhght`JjBF282h~)NV{wGhiJyAghTt#^IChPh=5_0by{b}JK!hj z$NO$odo2+=`FCQ4aTk)q{FO+)sZ+JAcIuI6sNsLbOlE-e8J4`XR zQ<>>e2_HVDzIiWsSBURBCx5CY*w?EhkhS`jSiErnkj2MZo~SgzaMmQ@O7EzNFEQ}( zN>jESs_m>hud2grNz8TMVbyZ&rU?9Otu}H%m6t97$z`%Euye!El01(iuYgWzSP;C+ zBureb0(Tb8HjdUf`(c<95NEp#OmBc*?;2qJE3~$|QbKJLIywG*yext-Qav^&vGGY* zryV^wT$ff85t<4%tXP+28vLVKEuDEa9rv>DT{d#IXjOc@`|Q-J>+Z!Bi?!RNmx2v@W*yjo58RgLs#c5pW9P*PN2fw!UU1X3(~w!zx0?hZ zay^y1!y&KxyS}9-MPR!oA~U`iQ$8+K{r-mh7hNRi)l}_Dg2Kng#(R5x4^Nps9z^*& zL#J;xT8YR9Eq(shu+hLspQZHbB_X6TE{*8(gs7So*b441bTu?h5M*UN4f`a0V!4Y#$#DugT%CA&?skLE4{b)!& z6$5{c?C~JGr&0NFV(UDyd9m3&8pogCeZSp1`DF669Zx5Y)}t~z$E*)NHgV6)O2ngQ z2I9o687nxT5l;?9JN(xlEy&qkdd-D5)r*nr&YV~%_;(_de5bFz+p zVAsO(A%HGtl{LxBGBI8}W1>M`Af4S@ye*%+)2sFv>aXS9x_^3SNv|4|kye(Ew5**t zu_&56;^fD?aH(wi8D>BZEi9r4a|Q6g2#AJ1>K=F7(K@rJk^L+E_3|yI0#?GVm?4(d z_F0V-34Zxr+xAOyRY&u6i~m%s4j%Khk84Pw0MaY-k; zdB{QNWI<_#TQ97D0IePa&42ZZ?F4098Y?(KAGH@tgxfOQG(6VWp%0%li|*>IrAPPR&dQ^FqfCOgNhRqk?mb5xZC;q{%=qTU$EZWzi^ z4IqvSe#kVVprB(L4W$e)RI2dT$Sdo*ntWTjF;n~JeOEaagpE1R;3 zu>i&h_V;K1*Bgk{gzVm-cM8rgBiv;h?S#SJ!8s-nqWw6$)_Iey*MyPwF%%zX_e#N( zU3G2nQ?lcJa|PE-b?w;U8{R0!-y8GX3mOD2)krb8`Te1raZ;^4;M0@a_r;!85yNb`mAx_Lo$qEMZ48OYjxRrJrn1{#|Q2e zFzH-uRk}y&@Y+IhEW=CJFxK(jtKO#1#o@MTIl#Gt*b9|<=P}Mr-s#5QiFV2u_mX3h zwbo<_Eo4&Y;FIA-T%g;j*TJdtJv&0{yX7QENP>hTWdNML&YzK72dLH$s3*b`z;rwqDQHoZG76%G5pHFue1bb}ZP^qz~X(r-6&R zDI0OAZdJOI`wZ1`C#gvExHe_N>lIz*_Y}E*2@M6*9?V8>XK@6-01RE>diy(f%{<>@ zuH|K&y^R0@d8Dtj@=P(PA?uUyemY->v~|Yu2W7>5|MN40P_Jxw$%zs%LjuNO%{BU! zHSbSj!2u~cI3U;{2GY-FvbD=Fa#C|?{AY9eQ$jVdRuc4VPa9E8UQcYk!FNbmEbl-d zu+9IZBR`H-Xbbs|xcg6uSWD8X7Fr@41`amDg8d-CM`#6|(kV!-vT=k@=cW#l;~NG~ zo#(~~)GZ}>jx;X=Z40l^Oq`Fl9bQlCZGzf{@lp9`}gy_Q<49#N+I+ z?r}lueZl8+dQNCiM7@t&w4=&>1M72de}}K4*YP_VslL1wp%%g2$qr3lE`91zQCw0y zJU&rNR8u`F`~p7;_gzi{NA_HYLljQT?ANJUf4`S2@(7f`WyiBWDug?Pq%>)xY9u(i zW3(=N4*p9mSp6f7_~jln7ai|HL`?~c?vO&tDDD>CTuvoVNN#02qxg=eA{U(l1u}n@ z8Bl^Y9+_26Gy3uRxz`b2>4(iogwMk8022A5(eJP1p=RG_6e?YxZ`iY@aD1XiIbJS6 z_J(}lwrbF-5RI-Kx;~Zb5;n3lJ}E}tsOhFa6QYY0%)(<$P5Kn6UT=4o`zw%4WrDo< z(&(pWT-r_s3~@ehHCe8y!D^3pDMD9Z$sc_pkfQjL%{kA1mu~NDJbcLvx|R?o+n|Zw z0?ue%L}20Ifv6IiDhDd7bWQLk_{(Zh%>GtFmgdkedt6Mymq|Yb_Ov3FiMM(eNN}P*A5psqt@dUqQ_i84+p}(6Nj%*y zH^ao^gzfMdV!P(ij2|jCJtKr21>Rn|ciFr>=A^7+h=gM5Pn^{41H~VzC4{mQb=(f? z7d}rp{6mGM)BCca@3d>CIH`L8ZU)}`lmPNRAF^V^oIbHKOi?~mt}m@Py_DK%W`3o>2E1Ro_zMoVk`fWr_bL{p7{HpQ%u6RZ^lsjao60iStQa-x;={FA#si z+fcS_w-Kny5+UTs(43f5CbFa~9YNkQ^mpk|qbpE{XT+P*`&cb4-b_9=ogjB)4gTCc znS~e1DaguH?0s({N5{8HtQ(w@;WH9$F~;?}uYKwXlK1A#$O&uRgbcKABHo8k_pBmN zl2o*6hbw8PbxKyRgDyx;h^KwD?eq+x0`ee@SZNSM{s({Wyb}bJ={L4R_H5A%d?B*E zSu)=0d%;97lGNKhJ!89RsonawEc{&R*Ygx zxv97=x`wD`!w;J?8*zCZ%ig&5#w@Jh>9! zzPfaQh$(;MW=*gDQn{*Hf)Uv#x+>(ay2-Jot&ggr{Jv8%mSOf}&AfMoNQn7C*r2h_ z_%Uuu_SdYHMh~4w_C2A|>+i4pxu#zFQKU99W~v7kE8h3(thK0$amy*hT@=+JUKVRF0h-4{ zz7578bJX%ee{<|=y)iCaNznNAQ_Qu$KZBlm)yT-Fz+{KHUxq?2h_pM#8D#@^oYahTLM+Ah=p5TaN`>|o zg9^LPw!BZP{K-`r#o5A}+t^n{!VXhaTOkWS&SPpQ^KVFH(TT7Nzh{Hyek^rWT$0}* zJ<|(B8)+;ZS^4atgrky)3FLl4W%Z=t_-Z-@dHfY(z( zT=(q;*r4&c9KSC2*N?r*awEQ+JdLcUg-?krDvKnD5R;9Z?0`dbJ!KT~`jK#nC&a=B ze-ND9jB(wMlz!!eJzCEo#Myv-_+dyp_9|o8e%qBTJ*NHY&pu=A;N{cWD3oE*n>5{0 zx6kk@6+gb9^EmNL+T(KlD}S8nyYqL$9;S0njQk5+R4`m~@Of&zw0Q*8t)}*V>Y?U4 z?QlB>7o`$7Q*<;6=IQNBe0M2>Pm{?I97_8G*+8$J7l9QgJk7`=}VdpDM%Qvk(|LJ4ercSqG-Sv+yZKE3IR%wxT3rbi) zvr5prI?BEuW-qjUgiqBJ>$zj1T%c zRVL-d#YO_itEv%o`c`VYi4asHa7iWZRR{WPq3wba@?5K39?4JBX!w!Y93NCBD{daE$vA7uSb^*f6goLV zFNF*ra&%Pis2TCUH2FrmZNEb8cOD*7c{L|i9B=$4;l9*^29&&OoW&kFIQiD`$rCAM z68e3YE>7j`L@IAHzUAkx1(75a{+0Lk5$)U@w>4F5d@|rtAb5 zZse5s{b#j&NqPq3XYp|1l?-w(3-Kf6Wg~d-06qJrZ+la5!86Q z#pboYcTiziZV~ERli7a8~FdvvL*p+u;!s#;~J`uvAr&1xmKuGsT?cdl~*T z?Bzg}LYt_L`EMU_Pi}g`gV*7+KaN|W{U(pn!l&{Um8DzPL!9#8iv%7h&`=xnF&d+%P>)WS4TvLK- zTLq+kR+hcr4n-JQNaa9&7?l;3wEl3n^c32nA9nr356WwljGlWj&Pv>c;E<_x8&^W@ z8_Y-*D!m$h)X3wb$vmR}Bigkv887pG$s!#@!8)vHcyy{28aa=;z=a>#dmik}Jht=@ znyjz%v_*U}Wtz@kaO&AI5om*a}l;=-9yhf-*66+V9r}#0%YC zR(ZT$X@~W58y*Xurzb0T1nYf+r#_<{_2^v7;Jx8ble^8G@(DiSMS}1&80jx|8?8kb zf$thR8v$Q@A7zzX6;wJlhmHQ23>Rerhg?06c_8&O!mftX@|@Fc@WprY!&9XwUD&gA z+&-$rGqnJY+~RUjYlDB!h~{$#8hO)?nQKFNo2Xau0^a|}(^oh&^?zX#B8r5HfJ%K0 zP)QN#QV|tNg;65~M1g^H4ln>|m68VOjWKGB9@5exqee)_7(K=q8}G&6`@Vm`?!EiO z`JD5d=XpMR^~oYfu~TFbKbP644RXi4T*aPSWMpaPQbw^H%be)SX1w7iu-s+EBQJNF zP&kW!JaoRCo$%A^>h-fHqTQm)^@JDLnG`g}cid-*peVjqAj4dGeR=Bc%aJ!wcJXHN zG)l@hY;Xab0FA2R=%bz%v(QQMRl*#*)=bWbSQdWt@ee*LKGA+w*6@r=SHG>*pOM4B zV`y+9GX1#d6ZhN`)~bsW5xn1Y;kvDu89{mN4@S6$G8LZps!#*(U6Tg{P$=LmZ{e{! zK_dM;%t5MtD)V@Ofg1*-kzn=>1v|$HpH;6DJ$LPPePLJgPp32f@zi=WdoVLDE8nZg zlhI3^?zBh&`=$x5Te4Nte+FQiI)OgOy|6JhKND+X9JrNW;)Q)L+w z&*;DJ4by+i>&^Aiq5v&46cu_jubITrG(p)K6&~m*rKs4>A#Y}QPTdw6a1nQ%+$qj^ zRDvv96M+XwKQR76y!uz$`)Dw2p-a+5GDC58FwK5?ooTx^iDYbqSRvz+8Z5Wj7;SGt z`zFn-Ui@hosNeJOW4hefN8>jS7rJfu{{7lDI!3NnaEv$AHE}%0xa1cswhw((EGzTw zqsJf~UWwZ>j}{F;Pb&|;7O&YwTevlUL(;+;gQ1Hr32LJG-EP0>2@F1G8l^rj#o<1K zW~EDcr}xuKGmn(~+&9#Oj65iletDp!yg594%%%%e!cvGBs1vUp8y$1;x&L;Jl4p>d z?h!soqaK6A`^tZmlKA%H%cJBj595oc4F`iy|0wXKP_*Tj;w!M#yk}uQX%^;0l@G-X zjQ;}XBiS+QkI$K{!z@9gF9OuWFl6$hZK>z34ucgLfy;ML z&HbsRuKV+JC4prLby-s{-%RB=b=%L-eDS+Tr#3zv=zN>h#~gKH(=1h0H!^5(xpDMoj>|P`FGFW=u9QY| z@9C-bG!~k2g-ObO;@@|rre@=82pnw6d`K;(EOw$=Im@%7>4PKeP+ttz`~U@{-jPLz zibD0rc8S41VMu+?NEM*F=QFO5CVg))1W zXuEbE32S@P150eo6fveM)QrhCxIye}#`RxUZ6dHX7FR6mLDVTfZAI#SwQGN(V4zdx zispdgnZs^amM8y6XX>1C6gNcfilX*sBYb8I>^sKq$n_ z;d4JYmmIB8ld%u({!ZL~QlVR;%s&yNv{AX4Y(bhf(uY**jL|GCKqtG=n(eMWJ;*ZH z9^OQn304COVB0*-0n$v`*o~cJzheoBl|mT>-J9Nv6R){X8!Ymmn);Lbg$cheHpOPm zF<4=$MxSW8n2ZgMVf=T3yGC!w1EEq4EO zszz@2=5w4E=F`k!8DcPABB1gB>Qgy%b?T`8iSq!&6%*t1vGSc-1U?vhYW2q)?l#l9 z7{EL3j7GgO4QMnLo*hk}kGi=SDHVGhr+lW#AF-YXT{1^-Dn+t%sSz zeSl2AXQ0+N^ScEr@9r|!wB2pXFD4HMhEnT_j32i^=6KmO(rS_BJfl|rQ`bJtTrKt& zy$`a^z6vaxIQULo-U*REKI?^~>*O0Rg_Tb>eh0*Wo{k3-Z+Ok~3m->C!WC35^|XOQ zlV*lUDeS{G8SercR5${>+)HH-ruPxIgsN5LTJ(#=PFYRqB1Y~yI6{z^VI;3a_|8ap z#=%kO9o)0Re+>g#HwC1H@^3fnshu{0`O;`ci|;MV`OP@a?!KpO*?NDAk>wXI;sFbx zf_;p!Rd4^Cdf=MNballBs2P5q=6%LBKH`9L?Dz{mR6PD}x|5rhdEv!-ZN1mAQ{9fZ zo5qRvg`dc~;RMNz&gwtMP+It`4=GNv`#%*hDHd6{FM%uAL^b}e>GRXLX&lSXV=-8- zAFiw?Su+3-Q$@8|l;Tp=>nnc~I^5tlC{)E(qvWWja$A89N^Y%O99^~X8_xV9)wg-# z%1w^mIK`V6PNcW`$zeyHE!vuF9FA}bc>KMsFdIE}if9MUAq)spY9lkD`=*NVAp%dB zF0a^tf^MZ256G{4$^qgY2se=c%^wPeq?#%gJ%_~E-8AxuQt=V8-2C8?WT+g4TY`+( z0GdLh#^s#1L=xOUz%_J+2U`Yza%Mc|G+&rytw+BnSIq^H_PRR9VUI0BmxE<3q;r-O zp>S?-XE*3VCipPy(eEU}fRtfAwKkOA^x_7O$o!LTSB(Wh_3%bBcBTE?4~G@A?}qX; zUxENyJ{MhM6wBSg)^(+I5TER;kDIn}D#FcwJdTCJgN`_Mj#}EUErda0%%^0seuzm$ zb16tyAEcg7jbZuG?xp&d92FhuLEG9FMmm9l3;QpN0O?fCuKz>!!xiLi4secw0^5GP z6tT%R6!?K%%o%b@m3UL%qr<1XDB2vZH=*sV_BS|>Q?J+bCNp$r$w=QSffe;!w6^bV zc8_r&k*MKi#?cnpS{m8y-!OBP(I^r8I>-{(gE4g8|BWzmDE|ktWo$w)Fs_`Ww*p>$ zGNXpxLj?#L!^niX)t#s5GQ&*ZMsHb&Zg{5xt=Hlap0hx)`ej{4|y+g`PCOsc; zK#9%_5q3TZfz-{IV!=d3Q7Da{g{rl~0ZQ>xtIH+%;Z}*G*f1NDoO@}%0Kw6k{E^wy zO`kMnndRCVE7!nRYaYs^n*6V&MNun?uD*yJ7N0;J0zp@v%K6Fal=o*FVD-9$+0Zve z(Xs9o0rp0~a;6tqC!d4hNgUrYI-5q=$|LFJSS@!5{e8Y!y`kgh=e}{AC3Llmo7e2( z!>~<1X%&s^U{O9w_{^aee%^JCv_@=2-sATN745yh$6mif7>K$1MP!5D{F61M1{eJ9 z^SV#llcC~Y@f(S1$~)R;fOctz*I=Dr?cU)^4KL97C*A!qt^7t|?>on@Tl*6)=>}*} z7d2Q>H{ph;=_pGW zGNIbrB7)FACz{sef&PlZDnQS9UPzJ#iSHA~dqaQEwUA;3&iBtw6z?vq)*dJQV95ww zLmuC?p))RF=tT9jFS=#f%((EOwJNNTO3q%je|a}I$(4}PDAXhv0?D`%n|uF&q7&-puwz8 z%T?^rOiI>|^@+gBmpzr|s%Lxhu*SpNk&fG`AFsMoCBv6zaPQ4+pHQDb+%CDAYk%KZ z-g)VTu7OA!euiqua-LN$C_S96U)15{_@=82v?mmNM{zs92KLF903(je9gCtrg|9er zq2+h5))*ZF15hB;OCBJrSl|^VJt?z0BRZp;fAbAdEXwW+&uUL-jt^XO7+Ci%E4`?U zoK0Snw2YN=u1e+eZBg7%vTBSLR1M;ud96BwVrw44Tg*4VftJjDLrkOneb@04zC)mg zQEM(YqHvdGCcPiIQL;@SF*rW*dkZ)5cXgKX@#UN|lWqqVRa-NNEsDCIp-c0k7Dg8+KGJuKOiaLhDK-QGq?=~h6Hi1N5r2r-l z7!I`bpmxVlW`f>&mbv?rF@zok!KF!}Ft?J{J#4i(>k!M=GOzJ~Cp-b0w3Pyf9gY_{ za#i>7Mz(6}-JA1x(V=Tw`-u|~LX*Lx)F*%h$n~3UhU|8x^3mZkycH45+01MOZ8}7C zd5U_HQTM(nk4j}t$#OI|_q@PD-@_HZTr2{JOM|ws0~e3P(a$#IF-qBJ7q6_jc$jwX zDEUwZIY-F`<$Bz3!E=hFD&FKC9YAJ?wXRM`iXjNs$l^9@aJ9Me4<(bX)jU6PHca!_ zMp3>lgVVi2$1FC`XDh5+$xYXff(RclT~^D~tTuoT?WItqM)7t)2fMm8_y$nFYL@Wf zg4f%Y4Ul8niuNBjvCC>hBNP0Bq7X`~daWn6q0yq@OyjoO5h0A#N~<*Uf1oOz){jV# z_5P{T{v7m|SFV8q3&%?-`P3^JCIMZd0C?bZOTFw7W2II7RmC45)F{N`60DhK>Bv?h zb$m-H&QvQiljK@U`+$jLN7@`)MZ{-F%e9Yx7t=OpM^{4z{23~dvQhjML>7f(Jb2++ z4X0k8pCwS)@N7?7Q-VXWr=HqGq zbN~CWWWZNTk2~7{+~~?SzB_1q3vWObcxIY=rp~nZFcrOBN8!n`WvV(8s&i#KN!5=! zXj~1$NE{`>FzwvWs}pPWy0KI8W3U?N*J9p$5obK*iF!x06^1KF37)5l-kNbA+wiGY zJkxujG63|BhgdE6|8aJTZJdNBcfx@b*Z!x(70{Je_ZICvHVZ1 z@omm07Nm=XH=e=5G zEoR%4%gP#%hkF|&jFGBLP>ttW>Mu-ueeglT#QvRgPG6&z^_7#{tiqSGKOjpPw2Cgl zC(8X}y{9&lB_=gP^6kOdaCz1!**DofK+z`fqMaLV;XSSqVs8`ko=u^Y>qDb*Tdn7J z`+QUxtnErm)@HhF)MAV)6a+u4*QY7Ey5R%Ve%GG)$TrhmGD>@{c-CzA(2Bnf8e-|N z0(0Ne#A_99&}RSaB>uwqEN2-dQY$4TlJgpC8%EHRBK)nMhw%4}$H}hKPe*t=>a4_g zL40BD|A}S$*mYdKL06ZWaj$m7%t2oJ1?O{TsVUbuet@$+Y6h!w482nNvM!&C*-Ye81Om8(6iMp zoX|oL=O-fymuLnZn?Z5PZ%LFs!Cd)LmE_ldGSw^Plkd*erJy}Kdr~(VA^~w*)UKrJ za_9ZNceJO$tBn-{4%638>?8}uvml_I^~wPw_bwiNnUmZ}zkT0fpW42&wJpU67W{a# zYHz7K3wvJ7!1=2kI2#huTcPT=!Xw$ed#;^7_kyrWR347s-VyT7Inf2Me9>GSM?DJO zZ=2;0w^CYnY!Of!P%q1R#}BK2f;{$WMF;kv)R2Rv8O7Z`6Wpb^JOq^8|JE!|j|LYS z1D_jk%#J*XTSc!Z(qKUwKVSM-p<*?ay4xJhv;);5{MkW&(Ol2ZAnt+(luea*moTw9k)YvXrSn1MlOP1! z1g)WTqVDy-QBMxo`pKN{_$Ig`vTTuJDHI@Z?=Guy|5d5mhg%Let5taf-!}KD$?YX9 z@9#U@a8p5WG=*x9TrP`DC!&bL+F}JPlX*nvaW!Dui<)^?u}4) zuZde@QrhYgw-4uNd=avUh=dF}hoABUtS<)_VKoJxS?(GosBrM!=EXQ+%Qmpfqt}@a z8(p=ytW*}d35#h$V1ahrYnX_$q-#-`v!v3>IeZshi4`-F$0(RoxYYPp*vLVoBuYt= z`JB{pEUjNpw+yEaCTx!Enl!SIwF=SVC6s0%%^!3J5BNs<^tYUSD2%rM8h~0Ov zUgh)o?PPY%lk05#rP@>GpITJq*1N0kkx)#t=fher*{cYdqF&Z_J2$meo0y0J4HGlJ z&09*YH<}iP)qV*ILUjYPW8*W0R5K4cR!kDPa~XnOlDQ%v6G1v2+8qF$@OGg#6hkK( z$%c_VWqkHyS;z)pO;=}*4f?(Tc(*W=Jab}ou6ncu$H59?^lQB!J*H->*u<%=*%3~; ziXf2|zX#$?(}z?>Gy0m{h=CbeHzzs#S<+(&f5YxW3m5TsV?FY$2ky))9`$?Fn`}E9 zUbROBzfatufJcKUDb_Ppuler|P62z12R@hiA@@nlMQaV*Uj#CN8pnIH{mdITE^8|S zq)0qAP^rq5V?VimqWJ2UMEk{`1{&Gsz$v{beXBuNk+?U>vj8`6C~Pf$jNI-s6jE|o zOxQF|Z^uQk9J%3g51`6{!cT+Tc!BO(qzFyBthJ2JQxcfrcGw8byxm)FFE*p2^o~-D zY2`sj$whcWsm|ExE-_KI=5J4G#1PE|>Xl*vVql0Wwa#ld39qZHLefxh+%USK!bF+$S%6T;7P-j_OY}!F$|eH7Q7Lqy>RhAr#@l^P z+0Re@vn^G!%$eZUMK^BU3IeMgMFQ16TB<(jjTM>TI#*ewzYflc_xg$G*;foi#I_>6 zR*NeaiWMZ6I)^;q6$|(;&QY7Kd#fKZ9Wz#OMa92^+=+FqBMAe1L~nYd0vOfV`aJjl z9bGeCz*kZ<4KXMXux)J$wU#=9o%vjfFx3m%C-mD~tor2mPT@Zasw#kA^_i+_!LIC_ zQ(=F6K+s+ZrT>}v5ZI+QX~9Wq_3Esh(+!zJMJ<88g;`QXnyN74=~9vA?#F;8`1*yA z_Yy`ZdXp2z2c0MmAHMg?#-Zfe9IV>S-(4-y(kD#}A77K~ay1arQRNg(NYcuvBR$<+ ztSi>5=^RR2z6_2rBk|%gyfoyhRgD4_>DsBd`11JJHI3y`bG5DOeH7=d%sZH_#iPB%AxM zP0sPEv~}-D*H^gP8>8+!z^JvJf{h!-n@mS3H_%^R2=ZNfAXhXoa9WAUA5f(*WGkKg zB@jw56U68FVY-@<1MlwGG#C10J+)(Q7(!6AncC-LEK8gz{rW4pGyv5Q^d`%5Z8zZf zA#I4|XPu7erjj{poyd?=6Y)iD3-A7e?r%vBw%YF67*AgSAM2q<(6M@+qg2`e@|a|i z9JZ?~mH@X)3xo)TIbtTWuf{gLK{W^|uG6O%k2w4?T$Irb96=0F_%W~Dj@d0}`L`A% zJ&rS~q$$nd^JkLP7lvdwcv*fDrxeoxU zoP2Hn>z{jEl9yQfdkY<*UfI%E*$axU)Sqoy01I}h-yMX&_N6}nfK4N?!joYW^_{1z zJ>f*TUOTk<;mvg_{`4zNzqv}YEXu^SlxQUvuhro{yEO4piv!OyMNJIr4kn?qMvnD2 z{tUQzUXDfmy&Os{CrtmG*9)pIQA*f~+Au^(Tmz0I(n*YjIjSi?jkf;FUN`zwb?TNN z&9j7oLTrl3n=@eNFyGWsno2{U6#V zXXx)cZc;B077Hh{DIJ z94-3|+HNj1??c&kDW2t-@g9QIh`^yp?do*KsTJ>|Nyf74KsTc9JY!i1mut-X>rXAe z{90o)mo9fxVy|6;Qzvv|EmOAws7%4}S(8=&vDS6ZjVhc6w0L ze@Bs)E_N0BWikstT$`STSn8v)xST^zdENr{O(<+%J)SPwa=Y#t;(TCEOq_-cHt2w> zMv%HEf}lHkzB-&0O!*Lq;0kFB%y;4YEv8xM^&MhANo#F~?7*c-DBWJooMQPA0PFfH zjhuWk!e19g*O=|7CMpoHE=4~GJ&9|n0PJiC^b#BGa((P8@IN&cu^eCgr@m-=gCn<` z!Y!ez9(pjX3Gj{h>!oIlCEib;k5FGjaMv{5kEp^fH(jT{H==(l6#>3r7fv4cgXyYC zt+z^*o~*w^Iro@R=q(*E0`!6q2%US2*m5FKKuY^{ZM=mhiLW7{5ra*#>9zOy7|SEj zl8_9$*3Av(qd2<8Jm3>y`>$p7xqvNp4nzoF`uo!`XD}ePZVPb7;P;pmT_CC}a$rWS zwxUlE5Nhdj1wqSJv+46;IxA7SqvX`B3$B7l7M&pketH;oxB(zo411|CY6;+9HuaW^ ze$Wf}U{L@BlvbcD<-?C?2e@;KVdt`bVO*{XTBo%qK0(RhcU+r@FUwCW^k8IZOWU$8 zkc!yO{r~XnU<+LgkOlbv&VKkY@A(+Z*%e(N>6I`<`NY8Ywb<5d-?ktvNCDWl8V6V> zzxg%?t5uoyB(1aa%r9<5eT<0!&AqznDJ@V)bel*7ch{*K^JMct%6^^(9Fll}a_CVN zpcf&KHFOk$W^OH#L!;E+U7ei4r~sivDqF8$vIfO^`GAGqpI_-T-O0kCOHv9de#n26&lRLMM*7w82#yxNh|c($ zHEdobU2n8f9U}UJ=XfpBEvdF1o?{ZOJMlTkxQI2#=-;pZExQ#J@3~z7Y8KCFS0IaR z?-a0f*+k{eQC?SUx*mdRLH_HX$ z72HWpZfZnB*z;~DsFCoo!gCQ_^9*s;1zJl^a{4tE;mS=%S7rx%FP&H@1%R|`X)O+| z(@hX{#D|y7I%fIpOtvT_lKp3?%ml4+e-OJ+vn$5~{AA%f0~_6g@P8=8S``M>;wXK3 zL>po?Ji@~7YHLXGXv%o8G{P*`6Q6BcRIPBJW$b(n&k9YXqYDvw%5<47gdJ!!>kR-$ z1UJ1b8>R_0UgPL`@HBKL&wV{iIc-SxS{b!qA!3oV9nSvv(4+P5`K{ zseJxJV5Sp5>)*NphKjy^gLHWAxKyMn>CQc*t9}0-_fws-U=m=8gQQQ=XJaRwMc zO=YCm?Zj+4LzkY)($Q(1bP_^Gtb(D%-yE`_Mj^28JLJN}TBa!MC5IUXLbXbFB;an- z<>xjE3AM|s;;tb=D{JMA!nHW_2&8ar|8wd{{d1MC`@V@3PYk$ci})&CX?A}K=7r52 zo}&qKP>;xVz92t*u2@tuZs7!a-1cOb6VWaN?sf z2_gV{>rXU1>lb60tFBAfzCHFdf!Qj>v*(5zTC@5Q%WQ25lGqpN|4x2{SI*3!L)UJF zYwe?)7JbQ?lR(BMplkyBWaT)Bryb#V79#&-0!F}#plPv;BfgQH=jM#*#vU=#8%1Y& zQ>NFtK+!SONwL^^pwF{7EGoy7tt$|GV}=hK5gWyB-!rl zO$LX;AX3BK%*JPD?1f{iyYq$_5G@eDff+j$x26pe+dX%<@i~1t5XJNbO0M!f%$nss zu<3!5$*Vm%(;4U~0k++46CSv$mR9X-R>_Y_oBFYlR4TC?P5$d-zuqSIyf8C;w?SKl zY|OBC{olGcncIIccn^7`a+{IJh$K#fgqe_6F zqp*;NepbjWeuU<$z;P`8svY=smC^VsW9;QS4}24CFS`!FLK64i4E|SqwffN`q_6)pW+951dgpAN4v%x|bffB-`jOYUBa{h> zU5=SE(yGJ-$cm}Nef&qgz#f1;0PjDJ={@@Ce1Rx$pj)R$+v=KOT4CZ)-U=~f1BiJ4 zvev4Tz9iOgJDJxoj^CTu{*iVY==DEj0$EWl_+USuKtCUDjmlYE?hhmyYp>M{7iN zqeK8}znM{?jx5kVOjMLab-u&TpR@>z+|a*%7+pSHXUa7LILoGuKj1nUEW`HdJ0i_1 z-lMuHq5zEglhBFNE>NYY_@^~3j*D@-q0fP)?7BTlwq$jr-@e~}?@P#m|4QJUx#PsX z?kr%hy5~45h=O4ULaCub>F+rft^*1gS9hAG8fSJIXxrg2;BR;u2Drz27Q;gS0%R`r z5EmlW=zzM9)vNn&+%rBU%?5CWZ*-%&IQw}!g;rJ#rJ*0C&FyGM+L$P&Pecr8Rc?@a zmXDugrq_2NL}LlqS3}RMs5GUSgA8-GX~|P7M%p2#%YK~v7f?Im%hEuWY9q9t3u*Pa2FQ zj0$vmNV~V!V!mun9)RiVVyNT#D3-gx{{2@%{65Vl7OXb_eb}#hNOJt1_zQjSB5DKu zea`$7t^l{z72a*aKjIWlKaGoD#6P#+cs4An^ioe$j5atCD4Zsk5$?|h*0YJeFCrpHM&ovwdC zPEjMvB^TiJzor|5*vh^S9y&jND7Sv0WTU=+5VSb9D;KW|96+7xa?)oTpP&?%<9_tF zXBE0FSG{?NSBPsA5{zZ?j1p;5Fl*16eE|C3$oa&XGDo|(4?T>#gySc#A#p8T?4>(+ zuQVMs)A3LEA;3i;D21abDC6Da#XHN7R+CG|UI9B%pz^pka0;2FnvoUXhHC$@{~x>Kd zUSh3$?lJU0(tRUBjVqh?mINpXxK!uoOQC-irZl8GYq0>?>kS%~KM=Wc zHNVBFWFz$z1ld-$u9n)g15Q0fh-<{rc)P|Tj&EetZNp|3)sc^x+&HU^hjK=5&dOIi z^MZsVM$QBix~+cMFHeg%ciROE8@fHX4v<3s)l4Ok;)c=={grEOoBHnn4-5s# zYrZdbF1z(rz)s(KKbB>?>i>1Y>DUht7<32a7A|uBc+Z6ZtcBhNpRPN}u3rJT)i;<8 z|6QXnPSss_L!x{s31m}#DG(>p5cH#xrQUQ2{xc}pI#_VjagwEYUYHga=!>2mDu(w5 z>jxcg&4Av>iN-t8MBU0Rv$Xh zOO(#_c18swvQ%;N{h%3txL5Icr=c&I>B+8MMxYH7a^eqhGoASA@mNnzm8 z09)Z1SNZA7D=t8$5jB2i5agT>Ta#2-Q8m`10Zy+y!D81f|7^~KS-w0WFs5@fnenye znprtL*84odSoun;LWKsf9E3VEj%8O9EJ#j(i4Q?%WIHwQ5IGUmN_FFcwZ;j|KFhNb1iZBev>jdy`S@DTTFjL+Ly;qQlQ9tZkB*VW>+2ne z52>2ZoRLQb0jI*>@d>K}Jff`_g54DRD0N~hV5<~3NkS#VH2dACZh>O)!|?fRc!8tS zNaE~*yl|7icFnf0L(pvfPE;1vG-UBtwVE33VE&~{$uU!%ATX=$APcR>y!Vp|Pl(Tj zg4^dVphjeRJj=YYyvlm#9o;n}$RZ{d}hfERb?Rwps}3P$ps|4geI6w#ZuTSC`!jxa!Hl+@dn6-!t5 z1(8$=2*6&-R5wz_9mq_)J<|a3Ox#v&3+P8rtL=8bNs@U~=sI#c^@IB9Ui0{}Z9S@`X$KyD?$f(34ZSp?aXoU*f_B5)&&d8(##|sAUn1@U^xfX9w4gx#{;-9^ z7(7YjXxbN>>%QoR@qX&#^d;sF+wu+pxB3A7uF22ac=^_l9=2}y0fG^=jsFle- z<~v+Wu3)5AOTWm6>EST{#8VAF~Q)JNm5fFP2C+xRF<3q(ypzWNj+GT zDc5NfFdFu^jNr^5LR+LjpQ~p{Ew1bTWmMpUkjkXdc7-~6a~fYHgBR<@>UoKHRY9(< zc{2yWPdd2W-N3uCPBB#5tK{|p+HHqGn>xClp}ke?(eWjG@cU`tc47YqpwUa?EMbbXs>LHP-Hba$`I9#2*NBQ@c&!WX!^JS_pGA^E7#%P^jB%Uo{ zGXsP}71OGf_L^`cON+1qv&>d^TQjeEx#hh1QRQ^m`(K_bd41!8;7xJox!nQt_P_~y z=|y<5u?NPe#J73}j3r_~(8{YtSAm0<5%7{10?Bx-8PX?>oahS*GLPVS7UK!9dIG(y z?!$e&jRwdUaF=!~Iqk5~3JM_s7R`RR5hty}CV964Whn<6@Z~tpv_C9W7qGCYf!!fN zJUjpGOz(uo2@)Ao35f zU^d5!Y$eE#{|;0p4DRwYd$Aw*N_;CB4gOv~o z57^q6{q04I;+Ksk1G=lG>^nvUKfjmrKy#3ckPm~hYIMyE;hvQ#r*~>AkA3u_WL=*eSc@xl>unP(N4_{y->anB(_7 zk!)G}qdS|Tx6pG>Ht2qV@uCwBhe zf_TgtqxRoYmjIBV@A~fZA!mL8V{O)v{Vc)Y&7j)9CE!yg7lACKWPJv>qf8pehNgjX z*#B7~xTJ`|R&$JdPPCtz_!+iw2rF{&(H&vb@}p$GzAB7ks7zz$Z+U9~P<%bE5Pt%P zXfCN90iw6O+%I!B=hgE+-~Wh|S@onFz6aHy5rfqt)Y{yqzm8Rp5_;2!+R9B408IeX z;Gb@oAOsPV)QgoZhx)IjNH2g#kF`UR;js8D1We)u11u8Y!hI1zKLiG-he!!O7C!M& zX!*2U(&)$w`|kfB;cXbIOlpk0+W#cNI^l=A!y`np2v;Dp9MR0I_ z6}!Aq;1s0N7y1VYCte$xQWa!LB;z_^J3vS1%h^KbI~jFz4OZQ3#;TWF(GieeZ{g*Z zqiLmjy0CKC903XT8EbAhbfWG!#Ay~?hI=($UmQe?Dv+$aB&zjAIOZ>!-aS@Oc#~C3 zuCxBDR=hJqsHUa>webm`C+>=pcZnev$olA36nE2YX6PJlr){0-n_-|Xi!g^zJP!IVKU@|@c! z%V5@tO+jLepqeG!9ybshY}f1iK$)_dT$ib}N0R{yPR2CF(inonMR(@_OTG`U@BMw= z9YBCp&mX-QCRAL17qrxc?!nD8W_q?5m2ey_=XJkuy z&@x&VtfPJ!D=oy4+tMBE4`VRgDhu$L_hwH(UC?GW(8~Swq)~dxZvPGs;!O9<(FFfJ za&9y}^_@XBdKAQ(kH)`lR?Mv@8uD?Kp?eLmkZo5}1f!Yn}4T^;@vetb9UAl)!2pYK^3Y)YOj)AWvS#~Sjp zCBOxG2wwb%EU`LTm-6EKT7^(RJi6r+0=OWDPL4@rPGbUj1wR$0lXb{TM4vN%L5;b~ zCj=Z0Jgzwi5^Mn6U9*5rNOluo_Yk~Cta$VDoMs~W?Q{AUq0cb|#^2V;*!$fE z4d2}2Ha7L{H1!!vb*CAtUeBnbMOyU$wHxP0@1i7G^|D?;r9X*lH67?;+_&A3tE0C& zJtP?R-txq<$58-Zo`hrQg60LPugzHUGjiX@y9&6X(Kbv(c6I2*_%J(L&qcZFSGc&) zb6Lp)>mCJwDIw28==fhSaScgms{kDK{;}U)#0%?u6RDg{Hd^sdySoqGSx6^q606Tx z!u(SM_X+KNc|&O>jIu#gM`y-;H1X$dWSZ(%*E_>!mG(qFSv5AvWr<9QG5vY<&HZJu zOy6Z$s)8KW%wEtAoV?kv%)QeOkv$Vf#rAAUnAK`w&2Qx2<$VM%Jh;h9GOnHDQI;&a#Tx{-q#_Pq-pua4Hy@Brx0ep2$M7A{5OBpx z21M9#WnJ<}zsY-1a>_@;uY7FhmWQACxlyj1yHP!sQRj97zQ61F!JZaMA=_T|;{BSd zJN0m%){Dynlp#S&oK}*;pn4Cf*}S2FoH||sWwsKClY2}PW@^``%v_opQnv3io$jzm z1*c!waC$t0Srrgz`ue5zxreXqL!ICezyhI%3VoI4JMjcTV%?9=^_f{#ai`gHo+91@ z0(N(}I+zSyFYg4@0zUxKYS~#%*l|z()FZ(A*1vWkU1_0CbR%BAs5qrW8Ui+ej|_y} zeI3{wCH^O+XsygiXyoTcb8a=j=LPU3=IEw}OMRJ=Hs^$1I0a%)t=N?r7QkBblOPuw zWnL;5P~{I+NT0TxsB;d%BAHq=)etj0Z-E_rp0{zdEJ*AE|7Y9tNtaNz0; z-;(`FB`}7?GvDSV?B2K`XKwsY6GV^q!C&5mN(9|xaZi5}97N8LTn%dM%Oj4t3|#u) z`euBjwcN1Y&B)QB|CNyMRL)1cJavu-eGPVuqN7Uw>+;Kh|T*K=8W?6`Twnx(da^nA;KMWM^^P^BZrDeKV#+p>7 z0?6*#p1t)=yvmL>f_Q?tSO2CNfo*-cgCk(o6KnW7fZfC+xehe3^Oq;5U?#`+f}_P0 zn6XCbWV$QNZmZU7aBMr!l%%q4?fmqeh-QrM#E|hAM^NUm#M=OORvH9k7I94>GcBUE zB3d$F!>*TAVG z>WLbcDVg^sSBRN!>HVhH{pRfe%I`HqNsDeWmY9V=&+Kz){f2buoZKo{ySCJ5*W3)s z=Nz@5ovU4+RB9#Pk73VLl$V{aEqb@Eb`L~`>~qqnyK2C^cYm365?X$@J+AgHZr-fU zf+%bBDs8P?d&ev+vg%yRuO!WDfl&$Q7x$>FKksQO`8P~`V2WDc6jZGsI#ANHQvpvc z10sFG1AYKx6L4UDjJ0`|KxJ|QELc7`D(-&T?3x@OD9TO^>&kN!@wXt%DciLpKBm63 z)N^wEbN_A8Zf8nh-%6n5l3VoL~=h@{w=f(yKehj`7GCt9j8M@WEH9zwEDxQPxn@Leqv{ z*@Z7U2RB`lt4ndf$M+!;y6(}<+>P7Q|{VMX6$aCUrmi6*X1!#JRI zV)pgRaUxhZbQFZp*~foq=a(^Xs`9Ye{QKiNwNgni!9x%8L|=S=b7A#Qp%%YBl?nD9 zsHh{y+&cN4Ds__(Utz7fk^+^xi?fv8M#(h$_K~NwpMxRjSd&(Psqi{9^bgBGVYQ7ca99w;7L= zQ!n=)1w73Y^a!3`dozeW5!d7pkJcc5k0=PuLS;0Q3MZPh&DHh#jh^gIQjC>kmKyQn zP;q6`7L-g-*AvD^x7rq?8Wg`I4akl2n4s<9fGAUAJZ$D#t5rsNXQ)H4UJEa?skG5| z>~_y=rElefM+Y+bqcWv>vEsuiMb!>(g>962=C-|1GY%C`O|I%Sl0x}wZw z%;=g1bI+Wt_vhi^w*i`k5(*UW+Wnn?R@<8oRX#PhZ@BS<-z|mz^e+M_CQevkOs4v~ zzB8iZ0V^L-g*c!HyGGt;BWd!{ge2WchIr374kih7B^KlV$o=P;KN?Q(O)ChpToZ)S z{25S~$!XX9l3NRZ0{CvM=y;0+XVkqbqA_3r06-*97i8U2>R0{e>2EM#y%4aIyQ!M} zXj2Qw-tIsFs=lIO802#*Ovnr*JjyTothC7wV(v-M5e#%f#n&1{M*_b=7ET7e_}Ya1ybJ4uXpDMsKJlb!K?c zF+$>G6~P=7fc;YPDy0H834d>IT7WCi0Cfmye`yL>LN0s)S}iN$URV9?UjSRLaMNyj zgh8a8R+cFXNgZy7lBSY5_+H&Ma;@1a+3aRQ(52U2)7u}A=q>$r*}qjDuHB+v{I1dM z+7S26N_Ty385KoKImjXXPS~+uhUZ-2Ot7J@{JQXONRtikfAOlXrRufNbXp~4VY_ZQ zQW!nfPwjHsHGdog#F5~D>;x-}AawKRyQ760yNT9bE{-#n$79>xwC(5kZ(aeOI~|W3 z8t$pLzc<6UTt#PH?STc;? z=;$^_Bug&UI5O=bV_+TIOM&xg5~t7`yAh2uLbuXszp8f?YJmF;`e|U_=&Cvqsl0vT zc9}&&oEZP#G}t{}T=u-%yc7j{Q$yvPDYLu{2M`qqV zjXyssfo19cwTf6O=>J_^IYZs>d3o#8-;Mjk60q4UDd~C|3MCQsX$``Si6#8ZXDM45 zgo`vgY6S>xIUk6-3hud&Zz;WA?lA+6I<)%P{{RQ-1Rian?tuKFa2=H*C{7CHU>SGM z?!e4_NrTee{1AR?c^9yk`3je$YQEu@ME}=G2Ll6LD)5~dy{I#O~-Op3hvX1Bo1qx7DTSga zLd>Daan6Sgn=LtoY&johhUIMLJa(}A{kiWy;r{je%N~#YaJ{$dy5868dYzunXYu30 zaQRrW2l_NiYIkEKgbh1vldTi$%s#ZJf?Yn(R|51y-3O4yzNjC;C*S7(rwhAJ{%WUG zBG#fDzbSBBvEj)J@P&c!OA#T#WHU#P$8}l(%&tV*>mXd|yXdl#E~ZYFHJ|@4ei-K zsEx8X4r^YzE55_xHUyDp#~oaH(Y;Sy_=gWtl77G&m*vb}Ms=%QyuXFNG(vlfx@?IcxyMDb|ro>t(g3tT&$WAR_mR|Xg0 z-C)|^?Q)%EXJldorzvPR#k2cs^8xu$Q{9JK5j4OOEGUW16X;p3J(w9-~=WxJ*GZdZv87f2wS zFAP(3svz9u?%>{0YMKyBukxocv;R(0wrWtb<)R^rG$_s14 z9C!CMK}nOcT666F4fQ}B@7R^Xu~TPE&DP;(ERj8Zcf5EU>)@d;eUGMhqytX}gx^<5 zaKAk`B4=8$Hb_>zB=H2WwvCjxVoROONKUYfzH5=oZdKOpZO^-ox-m++n_c)PN4}1o z-2Zib(P`fnVPw=tD>ae5g1Dn!OaG7hAr0g*BQ|7yNo2~1zo1eSf(Yx6{yP2a#$ykj z{j$I3fV+;1x_i$nlrngyX>}R6YdsMPWlud;9{+ZIHwX^SZqLeVwCrzX#1DT>7kzmg zz9O&h%-$N6N~AM!grEh{83)aAN8ys|RkwW%G8DB~aXybylx=O0Q!V;AzoECjGX*rv zjTM^}Iqm8rGedLF9V-nA401SjPnNzHTr~W_vG(Z~W@xlNlkENWVraQx{Tq|6f+6m_ za@nm6?-&kic<&y0?>b1VNMQf9`v=hL_Ds3uZ7E&yn5O{NWmIQoq~h0#T-HS(D^U>z z?ljISAY6 zXOx7c*AuQ1kKEpxlY^0Jp)5X#>o1a_Q?)?Fr?rmdiw^$mRP?jz{%ln(-$oQVm2?&V zVDhyO0NMR^pyFTlYRO&+x-Cpq#6fHH>oRm$_#9R?^HP{z(%Ebo@Hzi>?nlEqOmV^e zCB6z2agRi)-D7862w!d_WQHt;MY(sEGwTisR5AMH@(U{dDJ$)V@rcI_6uF(?{D#H{I$-bG|Z!gyFJ{tpioxYh#8TtvJ|oi0LHi*kx_t z#FoJ<{x2~?*8dAF#K*E}+C_5lC30i`f({VQXmJh>I-G0>nRe}g)@&BFonF?N(Uk)^ zuxjWYMt_$ggMTTVxe(*olsX~+HYn4s7;6;&;2^nc0$sY=BnJh08K<8N=r}nRZ!v>g zDl=AC^)dCh1X}?JCFrz*IlIF@?nw8ZmVE?>avK7v9=*5T19euuaPRr{TtXE5VE~|k z&M`Z7$a!C|A;7C97IRDDOz7k*PR~p4{({3g&t@)RehFY?RGAapkF}VVcHru{9q-m% zQ|?-FB+zRV4E}8fWVEf-8@2+s+i0ohITIO?t%_w8cT^J+p*(LD3&oK}dc(p2M-OXM zO<>2hD)=LlGE!OM5|kT3m{Xtm4aNLqIQMJNlx%{6-8JzZJ$4eJWoZv%Dm#L71+=%QRG zLnVlAOHi5r4aDmksh&Tf&BV8S_vMxk2oJnHN2$ z4)n~!K$*=5&Unn6VRXl=$L2qix1OWvvsXDMHgoob46UA5=8^zHpD*O5eJD(tBc&Z^|O^+mCe(#k3ei4E8(+}z&iX^-~$jfp*5(EgJ zb_%KD&bpACza{_>IiI&S^Pi6rwwwOJ9uiiicf~_fH;9}JrA%HThn*X(oqTU%5!trz zTV5zRDH-jR=ECLPzVO_M(x1aUxM1eqF(MBqUdjO^o#@}iACfXZMedZvjXlafpvN6G z4--jF0;9n^5e|~iC#aS{xra>n$e0NBA?JJ!UxI6A$U<2T7&_oHSoHWYXCApQCBYL!PhaCWYikZMoD#=^pPl|LlO0WUUudzIl3KjTXP8gap34jazjW7}2z3K3 zK1eLl5Tu5a3|hO1;mua?dCgQt3rW)j|*RNOXNwY7#i=q8iNL`GgJ>9ca7mba7 zxqVd?(YG&=H%A&%C(jnf(!3l0;r{cv`<}9;mPGsoFNcra?<#DfI}um%IJ;(DiUGU( z?zP_6J|n*$`INln7O$|olpH;?Y#8NwyJp$&{jooZ%jGCtT({3?$Q%QI7-H9Vvbw*6uYd4UE7UORArt%-km#Ns( z%!=ul#3n@msc1Cqf}`~ibVYx_d#m(_cO3%s_h83ctnNW-m)iG;2W&T$Ni93nWdI3% zh~h6tcl&-RQ9pdnVOIH%K+@2Iy@iLW3DDo3uT)y*&z~?FnjY7)kh+ddoth_SZLIx> zZ;?s;w@abcr+R4N$!Gy?8U!L%Wcc&&3KhheLcYPn>H*;;}cG~ab~C4wCv3lVL9KF z*Vqsqsh$;gYPK0xbqFdGD)r>KU=u}p(pV9uq1K3pD0RIK&`lqFB&Cm(j)J1)12Bmy zHw%C4kaO^$GaEZbZO!#NmxenO^k0N;+ll4Lf7(H(Nzk`jfYXJRR=#!?*r!g;TsoGk{y*5g(5kJ9|qr*Y3D?k{lspq3y6x6++_c zAWY6wm1J_JxvTd5x#l{ddfI&?16rh{T7utb!nl=pyV*&^iTYh_itNvrS!n69UI5c6 z^_#DZNfa>Rk68IEv%)T`4C5Ca38#%B-jCwh_|xHp?Xo~-@Cu=ihbh_~(77>_JckV|*6!c6G_Q+bF-Q)uo{oUG*HnzdG??x5j)Ye=$ zPlc$rv^cuKDa=hG>zS|(*`V_d>t3?a$X^#&=fyqUUT?SzPDoNh=#a#)@xyBZuqI`Z zHWKf*FRdHP^f*^diR_OVdSJRaZHLSs`U|=_cdJ%3WuwbiyycFa*o^eJUGD@=@t7q$ zQ6%GX$8l0(sE&U$(V^Ig-pAc5Y^n0R5j&Uc_koQlQ~)NgGK6?}<`c*8f#@=cFU@z2@$PO%?|OW;u)=$SrNA)XxY$qE3Xfi_j|JRAu*$0;)6rUWf#^J*Ml6M!AzYfj@C)o zO1%%c8AwJ=z4CcEo^I`pIpwZh{el@a)jG4e#6kY)g+2|_j^}T!q*rdQO>?mqL<+5^ z%;o+m*q+^-Y5CRjX^)QI%vY6R3T*-rzKd09Pai+Nu5?{HJ?lPW-m(A?8+s#-?m*Mf zJ$N|-0-U{VR6b5-v6?o>OgdR=2^Q&QLuoR=qvupT(mcTvqs)PcOhvq!Of+S)|8F?U zZSiV7Bn5+737HB>s?$rm6)G0MuBtVkAdZ7SxQ!qpv=^LW(I>~{`hTCI207s-IX7r7 zk69Vaj-!lS>-SFF%Zv4fVHmQDw!&72x=D-uf1<7nmly32HM1niBBZ=4`~8^pv{mv4CIxz#SW@kn z68hzhF9@$m_LO>#^l01o^>h8{bBcTnm23P;3phT^Ha7FkUU)Sy?G&czl!h_S(VX|e zociz=9S<9>YA}1jUkYS?Os!_l)fqnXlB~LZ>J+k0Hc_?kArJS+A)$)C-xE_&R~pwa zbf~B9t;8TXU#-db#+!|Z^t{~%9<(TI$D-%6gv)8VW0rriPMH3lQ|>doMo+z^v$qo| zF1NFjzc$jryYDy9#N@c)rg6Tst1tw7t(FxB*|ifO;*+NvzvX6t!`WPQ_9!WU#Ee`H zUi+KFR;^0My>dQ9|A>tUw8#M}p!ic;!arJ6BWn?gaF)QAmS{^_$ANxlok zGpaqQo!?~IZzue`BgL0n8gSHA5ZNjM-NRDurgj?xF;QJNNn)z$+Kh3T2|xcDFZZNR@4{^@4|d|86Li%3XF4RpcPt znG>OXovI(ODEx4P57c1{kE)b16B3gwl`lO~(6hew=*VciQ#n>@ zE)UOFV)<3pTRIo@wAZF~NR4jQK!#$DcvDx%D_WGHwq#?TOO;?QHyepB(E-_stLpG% z0=Xa3TG2GTrpEcQ#PFF@`qCAM-QYD6kHd?}e_4IRgSpSpUP5+%e*pKs9^JXy|K8v8 z%NnLPE->ZvYn>}X>8c%9onbbzDh4b!gX@vsE5>&RmG0kLdnUH1U|)AL=B$jGazYbN zZ=u}zk-m2znbTL4M2;PJFsQi>ab4SICKqu*t}s*i+vf@%Ifk{uDrf?AZfo`oOQynQ z)Mz&X&H{+L1J$5D5BvB|FD3fem}^8puLuPrZJDZ&Vlv#bQZw(T%t#(f9j+dl;S65F z9Jeoj?ZXyyJJBH`^ccPSW{>{aw<=Lae*+0SRSPBYy;Nwq48kkdx7eR0D%AK3zG6yz z-z^ffUjCu+@8ED-cm+G_@_`qy_m0w!CF00S813UWjxqqiH5x8EB zo=|h@`hi&2<$wt*N}_RgLc&qWJ{^M2mzM{CmRyEnS#nsvxh8+CkJ$dSb{4wknrgYP zp=Ztc6CEYNFUa)veX#5i$wYNCYju!?TsgM0l*JY4NH^FHAo#xzIDvsByKT|@=iHK! z)B1&(A+ETMriyqPhWC%(fVwT>@&f0yq0HGalOEf6d$v58a1qbixn&@Eb$RHClZw-Y~A*}>bRo}^)uLs|fI zGW34m??2vXE0McSb$)5$K9bEhMeCAt_m8(VJ-J@u-;jG=8k79w!+~c~PEz~SP`_sP z>8|sQsz5*UkG&1#*FNujyZGIMot~@B{T@zri2gyT_IP5Xhu^?3a}V-(=O>Cr@*JdI z&q`QoNXBQ7d9uB+V{L2Y7WHBEbcY74pk1vIMzq(TJ<_44v$uMc@$SNuf`yI(ey@(O z(BdN_*nK3RCif`rOK8azR3~L;WriC9>Fk!EfK#X`gOvQP^~+B^Yb;@I2&5}^ZQ~Wc zFL%Fas&VB@flFn1Uo5U^oiMCZ8GET_wwX*Ba-v$84@f>YB5y5yMNhdrvanWPhDsA; z)U3zblXcCJLAUcj1DoPr<}q+fY&Qix`o_5BqPb|R8;lbWv33=>yLXyqyH3#*EOlUF zo9i@^{7&4m7OS(tRV(3}&X|R?y0=2~qm1#AtkPnD9r@R8t6j2(v?p0O*;{WmFfqe& zGWh%WdIPL3IaVF+0;#6Pdu9CcG)Y*y=xtw=X?E|)&4RqD7Wxu0`p-1W3ZI$&=s*L_VIi0kLxx)RhKlB;^x))F`dzhYTWOaDrV6oz8 zsd5Ry{IlD$3#gHDpI1&Lt0JaHXSx+WV5pHPTC5K53=Vot-)+PQEo+4-51{%`{Mn1n z;ZJajNzdH!@HjSUI?1-lS2BVZq7~!H8|~JTDH&WK*`Ku4rG@wxz)z4F_ouAHAa(+x%vfMIy>u{@8Ae+*)T1ad-9Rvj=AsNevtJ3YnA72ug+c> z9`6~Nr8^`@?eiB~jDf@j{-FKPwTCC#v{nMrm!=gV^Zf=&<`!xz4hxE z@wC819kLJsBGpLg1A~rgf!uLCQYuEXndNNj zFV7la64la6Ps)Lm6TPn-vpmrL#(C03@W+ps!_7#AoUMk6K4Rn}wJNZr)!ePHw$O=h zi`v$)WTUtUn}rAi!vG@mD)Zq)A+8B^5=!2NuPvGq^G-4Yx>?3n?#yVkS#W4E6CK%!o#wdb;+C!G`zSBV{-qPm?l z7pv|M>0WY@@)eMh@U=UJc=h@c-vNQUf)i#a6o$f#=tKD*&Nrs&WC;xT^_BYNmiZUV zJ(-=Wy%o-@3o>X~H*z1zDiv(qT_fy7&9(Zfga!KPuCQG%tTfFrXq;!NIT$@=_!2qG z%{n^ae5FXTe_Lfwz}-l8z-Ripnf+(zm@NkD@Zf}hOn32m_=eo$@PX+6w#0z>UFx@D zbEJC7yuAIFEQ1PACK^*w$|qX($6PG8TVu(!bhX|J=PEz%-!ub;{OM-j-*r!46_P!Y z?6ObbQda198O{D`aoLjR)dtFs?rrdp&%t$GajCue+x`C6i=^cxGBB;>!A!VIJ=Mz! zg@sohq(tPD>{ttRkioCn5)8s27{I zvAiA3bXpE~U^+~MH1|qlQ$*ZGoz8I7T(hkee?LZt(QJND@pbcWfwx(b_AK z3hd#qzGVf@Y}ub(g3#waYUU=!Xop`U^~kk=Un-6BxuG4p6JmZb#%hOP4^T?O2b=&% z85r8V7#K}t2H>23mI)majp5E*QJMHQr*rM%FJfkOYtXwYsL*rg! zV!MOK6pRX|UPlpICYfdKSq0{VnV3M;x8xw0v3FYpI_I zTg?h78|n|j?!Ert{dv&kaua7pH4#JL()R^iKIr1|@bQa<{6HX~-e&VrQJ^@QJ)go) zF&-Toj~(?8jHOj#eTL?ZYagHesq=1S$XMl>WJKn|^wy(s%n0nM1nu&5sIxkLdNHjd zUS+3kq^tY|Ex6_^r+nMWso_Id{JLvy@#Xz%q0_~#=2XOV<@HjHypQn7e==joK^zNm zv&9!O3VyAmBdrByx+A!2c-R^?go0 zewq%+o7A-qw5@vxW2T=-p_D7X5d0L+z=mXjgz&0~{UKQ=P6f}Mj+BMsr}e7X+Xh#n zwpTah;s4U3))v#VZ5@d%e&}HwMLRcg<&$1_w1;=j0_n*~Q}z_KQO9C*H)q;zm7T+B z(PHpdBmhlP)Y#$gox|%rCFL8UqOR6zPDiT6!oHqvM|OA}nyX$Um*SvjBC-aeBY%N`)!L#lwS`-OXr+}wjxo(H8wttOn-ojtcR-uk3)G*M$b=&=SJ!KuZk->B5}7zxJ!#wUMF|5fv}`gKH+))L47eoM?m1 z)6Bh4wtM{g;`7fnw~XxDD3Q`c74vEz=nY(|J7c5recu{6@~l61d~1Wzk+g*+h*|Cy z*Gy+`L7&!K(*)w~ao48FZB~T*yjR6(+aPxTTjQGA%-(4^m&Tm4de&)K@2;ile>Jx`Qmd zHKp_hoib%Sszc!?W$9Wb6lSk|4&7Gb6dc1nC?4C%hEb%kp8M*B1}Sb*q`dHPzg!fM zNmq_PtNUhiyYa0)eOCO5`8v5qPd(3Q{&rwmq2qen}(7DQsHW|eJ93VpK7 z9di~=89_H`i?(%}o+~?o7u!{$#Se$$tgSRwG_fEh0?>l@E1xG>*j169{i+4PS&5*^ zZfe^ol&Ph+BWFxO`Nwdcr0g(vEpEic6OeB6QZG2sBP5~pXd|lnp47frpY?Lb!d9YV zVLQKSc<-&~iGfG#6&T#*Oazn84v7TC+l-YrV#mioW?F-XcJzPW`%AilHDaDuipdMX zSjc}Gd>;FCyzZ71m^MhnPL|fMmq{5NEwAU$180yE`bW?-$-jexlXefbfMEU+$4z(& zCL6FJlieM6{qyflVOjtE*8)cJFC|$CP8^{{JuBW?$$Umk7nlk><#ePOLq)JT;Uz%^ z&q-w;-a>Q(ohdzIIUA6S(&I>Ujt7(AVGo zM19A`Z_)9&jp)?a^|$ALm-{`*R*EL?Ci)uwQFaT|j;R-FG~A!9RCg;eIFVUtU&ri1 zsV3B9KYKCow-GjRO{ke5Mr936!!+bxV|_BACorDf7u#|($aU8>N3BeD^|9O%R?ykv zF=P~!1y%Vmb&>CY=({ovl~AR+zrjyJQ%*$nKQsNN#pV2YGyTLk4-BS0zuf?r9EZ_$ zAo}>-!r!0*kn`QqN61U#+q9tNpmtt&o8Py;>#9)RG!^hY(?I zr+qzLej&q|LXCWjdtJIXo|=Y42ZG>K#81)oLVf0*7J^OqaI%aFhk|zBStf36g|nB| zgTs;A&X9)aFzrbV+A&c1S<5bX%a=9$4@2OcKXd)cu>zBT5mCuuK3Yij(*P@+=jlwIOT$2PRV+Y(M&aD%Sjv?4xSZVXpAK3w zX0I8vqY_2BUw1kRCiyS&TduB5-3{iGu=*3be#7}sqDu5R8l%C<`56I94rfFoCiqDS z!;jae-9m7T*TGacd)hpWpgzF`S{Wq9?*7Um{O<8GgYwkcLl1wqtNo|rvXW}Z@ow4$S z&-{yEeuLm8PmgcR8#N}3w7gQBhw@Nfz$xw*>UHUgvGN<8El|0DSdCfm4FlM%dP0e_ zN%57au)PjtN92A67z&4EZe`wxYK{dqJ8%%NEY3j7EA=tTQP5#LlK4k2l z-tRUWwT(mmRp^_$*8QK$x@j}W#+(cdFIdg(H2NRfV$caN4WA@ye~j3eXGz(0TSGFi z*Au9FubZp4u#9(}Anxc|onCmnaY;-iSp|I4+aZq~7pNa-qakB+!}`q=B`$OAwH)Vb zs^{cepauDY&sE~_Cb5M(s?pObvW52}E;?|nQlM~dc0F7&q`!0sLkY1Rzk$UynvlT{ zW2@R}K8(jh>$rM%^5E3LBll8ZxRMPMY7o%FITZs02c>&nSg!(f5 zTSNDh+vl?XM9!oox7kfVPKk6~8DAk|pnK0WiJ@w

UfihaCQirj4A9t(VyJ?^5ek z4ZHN@8L}%#vYujuh8w7Z)!H?E-H0tAx&@l?KU*v}MoVK${0ZgI8*l#Z8k)t%nvlPj zFt#*tcL!mw`xw8(c}ovx!BhD*^SOPx)={xPmOn=gTEk!E0qv`@>Q?GIW4;x7^6iJ9 zXl~ZcGxLV8b2A>jAx*6x)9&yUGy=*1CvIOL4}bJ$`z)8N5gdQ4tmukXo4#u7CeSl} zj4HEF3xP^Fvl6tsgUp{=y!^R?0*v+Ui4%;7IV=qirAV1BvCv?u>vg56G>QLy7J!t< zf~~%qD))NlDrp#LhJ*-KI<|$36UPtz@jskbdL2hNT+v1DAeSB)sA?<$$t0di$fbuB zvCwG>d62O?5o5&UX%>^9js{%0fO%j7)!h@)a*YcuZ9ztGY5$lVA-59zkKTWQB{#Xk zS9tcZsl%qBo0!+|Y|2`Pr(AQg8T1VLfL*qi4o%6s~K)I79= z>KyYmPY>71F_5+>EbaX$-QFQsxP$%6eiF_&I^61wwK$?d{h&EK?5-Q*1AIqoiFCitB0nI#V#VzjtzF}mvg+eD~`jM8A@XrKms_g z6Sdyq3#$a?F3ff^LtDpCJ#_rjiP;@bdsia%_(_Qy&f8I1EA_Gbv1!Z;0!d?8s<_pk zVc42rU?Ee0h1{yWD|d!lecQhB^j{!oV;H_XJywME@#}A`bR_r5s)>}l4~C4NbW|#< z9Z=DGOy&0KaNp-7pr|3~A#<@`m={preV`>QAK0OU5Ws(P|1Qp$(_@4Irx(MI^z1|H zs;a#Ac;+7yGW&~+%oDD5L_LqB7lB`Z-RsAVV2=GnmaOTGyRbo8e+W19@%oZf2hmYM zz$k%v!T97HQog~2x(^*y$2W4`%6-bznU!P=*06sxB=5XZfSmWjxAuH@*e*cQVU2ds zBJUUoHa{?zn^7U7JFY<(Un%rvkRjwvXY=r|5hwgZTnJ0RP8+Rd@V0ym zi5RI{`fT6E1lRltHwoBWN0h#fQ>`34{NqRJTr2z97ll8iS=w?Wa;;kgb~JM%WNv#d zW$AFCO!jb$eSW-gO?78wBb*-bvv`f=99cJsZ3{IxgFNUwX$SECb-ceRsg@ObD!G0% z<}7x|HW-Te_R$1V+Bd6uIt^rMi!Lxj;$wgC7O z?`lZ;4#0Tr|HhjrOkl=?Odp5S>&DbX<4d^>P{(ZH!AWCb+M9TFxYY0^(a4>^wZ610 zIaTax#8S)|T3`6J=t-o1V<&sg@qik#%r|mHMTGumBQ8ML zRN-!TH_=g-b`)Cw)P_!x8R%fI&FA*W6aiId>r3G5#;SKE_mO3fYc$~uBMx!@>O0+# zRvFK1HPp2x>)?0VjE)O)t>TvNZxFQ%>!gYjfKf{*XYL|SPbcf8?(Rgyl|xWumR(`h z8_&$Z#>L|Tf4QMMr-uo0q*cVh5+L``>e@bbQ=_+&`Av7~jrjEM8>Ht+J-0Rxfm+TcR6w#L@ zzig~9y%7Jc!N4XBFLM4;1jvC;tH+ z*78+%*U8zL5xIg-IJ9uF5fD=!0<3>dq-ENa9gQpa4?tiY`Zic$!W-nKbIy{A?Os5U z<*9rBtSA4qmRT&hM}aJ?qaozDWIvZ?w%Aty@5sl;2euOeK8t{IH^6fU5QG9>N&tE@ zKEC_>`=kNBM8*Hu8TqUM*O`y+>yiIAui_4f)@S{qKeO-+B3;x%i*8{NH8q hzq|7P+g0Gn?I{RpTIF)D04@ig@m-5M<+oj;{txg)_F@14 diff --git a/.well-known/ai-plugin.json b/.well-known/ai-plugin.json index bc08de0d4..44e8435f2 100644 --- a/.well-known/ai-plugin.json +++ b/.well-known/ai-plugin.json @@ -2,17 +2,17 @@ "schema_version": "v1", "name_for_model": "text processing tools", "name_for_human": "MetaGPT Text Plugin", - "description_for_model": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", - "description_for_human": "Plugins for text processing, including text-to-speech, text-to-image, text-to-vector, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "description_for_model": "Plugins for text processing, including text-to-speech, text-to-image, text-to-embedding, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", + "description_for_human": "Plugins for text processing, including text-to-speech, text-to-image, text-to-embedding, text summarization, text-to-code, vector similarity calculation, web content crawling, and more.", "auth": { - "type": "none", + "type": "none" }, "api": { "type": "openapi", - "url": "https://localhost:8080/.well-known/openapi.yaml", + "url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/.well-known/metagpt_oas3_api.yaml", "has_user_authentication": false }, - "logo_url": "https://localhost:8080/.well-known/MetaGPT-logo.png", - "contact_email": "hello@contact.com", - "legal_info_url": "http://localhost:8080/legal-info" + "logo_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/resources/MetaGPT-logo.png", + "contact_email": "mashenquan@fuzhi.cn", + "legal_info_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/README_CN.md" } \ No newline at end of file diff --git a/metagpt/tools/openai_text_2_embedding.py b/metagpt/tools/openai_text_2_embedding.py new file mode 100644 index 000000000..822c5af00 --- /dev/null +++ b/metagpt/tools/openai_text_2_embedding.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : openai_text_2_vector.py +@Desc : OpenAI Text-to-Vector OAS3 api, which provides text-to-vector functionality. +""" +import os + +class OpenAIText2Vector: + def __init__(self, openai_api_key): + """ + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + """ + self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + + def text_2_vector(self, text, size_type="1024x1024"): + """Text to image + + :param text: The text used for image conversion. + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + + class ImageUrl(BaseModel): + url: str + + class ImageResult(BaseModel): + data: List[ImageUrl] + created: int + + headers = { + "Content-Type": "application/json", + "Authorization": f"Bearer {self.openai_api_key}" + } + data = {"prompt": text, "n": 1, "size": size_type} + try: + response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + result = ImageResult(**response.json()) + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + if len(result.data) > 0: + return OpenAIText2Image.get_image_data(result.data[0].url) + return "" \ No newline at end of file From 8aa30c35d2da9345a4d04c073d38abccd08d5f63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 12:13:52 +0800 Subject: [PATCH 043/378] feat: +hello.py oas3 --- .well-known/metagpt_oas3_api.yaml | 87 +++++++++++++++++++++- metagpt/tools/azure_tts.py | 2 +- metagpt/tools/metagpt_oas3_api_svc.py | 1 + metagpt/tools/openai_text_2_embedding.py | 91 ++++++++++++++++++------ 4 files changed, 156 insertions(+), 25 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index e6cf25d86..4999bf38a 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -101,4 +101,89 @@ paths: '400': description: "Bad Request" '500': - description: "Internal Server Error" \ No newline at end of file + description: "Internal Server Error" + /txt2embedding/openai: + post: + summary: Text to embedding + operationId: openai_text_2_embedding.oas3_openai_text_2_embedding + description: Retrieve an embedding for the provided text using the OpenAI API. + requestBody: + content: + application/json: + schema: + type: object + properties: + input: + type: string + description: The text used for embedding. + model: + type: string + description: "ID of the model to use. For more details, checkout: [models](https://api.openai.com/v1/models)" + enum: + - text-embedding-ada-002 + responses: + "200": + description: Successful response + content: + application/json: + schema: + $ref: "#/components/schemas/ResultEmbedding" + "4XX": + description: Client error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" + "5XX": + description: Server error + content: + application/json: + schema: + $ref: "#/components/schemas/Error" +components: + schemas: + Embedding: + type: object + description: Represents an embedding vector returned by the embedding endpoint. + properties: + object: + type: string + example: embedding + embedding: + type: array + items: + type: number + example: [0.0023064255, -0.009327292, ...] + index: + type: integer + example: 0 + Usage: + type: object + properties: + prompt_tokens: + type: integer + example: 8 + total_tokens: + type: integer + example: 8 + ResultEmbedding: + type: object + properties: + object: + type: string + example: result_embedding + data: + type: array + items: + $ref: "#/components/schemas/Embedding" + model: + type: string + example: text-embedding-ada-002 + usage: + $ref: "#/components/schemas/Usage" + Error: + type: object + properties: + error: + type: string + example: An error occurred \ No newline at end of file diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 6b1a041f3..2ec1539ef 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -108,7 +108,7 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key if __name__ == "__main__": - initalize_enviroment() + initialize_environment() v = oas3_azsure_tts("测试，test") print(v) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index ef3347b6c..aa5f50cb2 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -17,4 +17,5 @@ if __name__ == "__main__": app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") + app.add_api("openapi.yaml") app.run(port=8080) diff --git a/metagpt/tools/openai_text_2_embedding.py b/metagpt/tools/openai_text_2_embedding.py index 822c5af00..eb90a1ea9 100644 --- a/metagpt/tools/openai_text_2_embedding.py +++ b/metagpt/tools/openai_text_2_embedding.py @@ -1,47 +1,92 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Time : 2023/8/17 +@Time : 2023/8/18 @Author : mashenquan -@File : openai_text_2_vector.py -@Desc : OpenAI Text-to-Vector OAS3 api, which provides text-to-vector functionality. +@File : openai_text_2_embedding.py +@Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality. + For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ import os +from pathlib import Path +from typing import List -class OpenAIText2Vector: +import requests +from pydantic import BaseModel +import sys + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initialize_environment +from metagpt.logs import logger + + +class Embedding(BaseModel): + """Represents an embedding vector returned by embedding endpoint.""" + object: str # The object type, which is always "embedding". + embedding: List[ + float] # The embedding vector, which is a list of floats. The length of vector depends on the model as listed in the embedding guide. + index: int # The index of the embedding in the list of embeddings. + + +class Usage(BaseModel): + prompt_tokens: int + total_tokens: int + + +class ResultEmbedding(BaseModel): + object: str + data: List[Embedding] + model: str + usage: Usage + + +class OpenAIText2Embedding: def __init__(self, openai_api_key): """ :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') - def text_2_vector(self, text, size_type="1024x1024"): - """Text to image + def text_2_embedding(self, text, model="text-embedding-ada-002"): + """Text to embedding - :param text: The text used for image conversion. - :param size_type: One of ['256x256', '512x512', '1024x1024'] - :return: The image data is returned in Base64 encoding. + :param text: The text used for embedding. + :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`. + :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ - class ImageUrl(BaseModel): - url: str - - class ImageResult(BaseModel): - data: List[ImageUrl] - created: int - headers = { "Content-Type": "application/json", "Authorization": f"Bearer {self.openai_api_key}" } - data = {"prompt": text, "n": 1, "size": size_type} + data = {"input": text, "model": model} try: - response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) + response = requests.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) response.raise_for_status() # Raise an exception for 4xx or 5xx responses - result = ImageResult(**response.json()) + return response.json() except requests.exceptions.RequestException as e: logger.error(f"An error occurred:{e}") - return "" - if len(result.data) > 0: - return OpenAIText2Image.get_image_data(result.data[0].url) - return "" \ No newline at end of file + return {} + + +# Export +def oas3_openai_text_2_embedding(text, model="text-embedding-ada-002", openai_api_key=""): + """Text to embedding + + :param text: The text used for embedding. + :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. + """ + if not text: + return "" + if not openai_api_key: + openai_api_key = os.environ.get("OPENAI_API_KEY") + return OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) + + +if __name__ == "__main__": + initialize_environment() + + v = oas3_openai_text_2_embedding("Panda emoji") + print(v) From 34d46829ec62bf5b41f23cca5d566f2adcaa2f20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 13:43:47 +0800 Subject: [PATCH 044/378] feat: + server port --- .well-known/metagpt_oas3_api.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 4999bf38a..7a0058b50 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -5,6 +5,11 @@ info: version: "1.0" servers: - url: "/oas3" + variables: + port: + enum: + - '8080' + default: '8080' paths: /tts/azsure: From 2b19a7118d54420f689f98b69c85fe98c8e3417f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 14:04:23 +0800 Subject: [PATCH 045/378] feat: +servers http port --- .well-known/metagpt_oas3_api.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 7a0058b50..7ae10579c 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -7,9 +7,8 @@ servers: - url: "/oas3" variables: port: - enum: - - '8080' default: '8080' + description: HTTP service port paths: /tts/azsure: From d97231933fbb02a19f8954efe49679fd8eefed76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 14:45:14 +0800 Subject: [PATCH 046/378] feat: +async oas3 http service demo --- metagpt/tools/metagpt_oas3_api_svc.py | 31 +++++++++++++++++++++++++-- 1 file changed, 29 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index aa5f50cb2..34ae6a563 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -6,16 +6,43 @@ @File : metagpt_oas3_api_svc.py @Desc : MetaGPT OpenAPI Specification 3.0 REST API service """ +import asyncio from pathlib import Path import sys +from time import sleep + import connexion +import threading + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.utils.common import initialize_environment -if __name__ == "__main__": + +def oas_http_svc(): + """Start the OAS 3.0 OpenAPI HTTP service""" initialize_environment() - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') + app = connexion.FlaskApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") app.run(port=8080) + + +async def async_main(): + """Start the OAS 3.0 OpenAPI HTTP service in the background.""" + loop = asyncio.get_event_loop() + loop.run_in_executor(None, oas_http_svc) + + # TODO: replace following codes: + while True: + await asyncio.sleep(1) + print("sleep") + + +def main(): + oas_http_svc() + + +if __name__ == "__main__": + # asyncio.run(async_main()) + main() From 3c93573f93cfbbbf79a523dec2e3cff5d2e719c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 14:46:33 +0800 Subject: [PATCH 047/378] feat: +async oas3 http service demo --- metagpt/tools/metagpt_oas3_api_svc.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 34ae6a563..277d41dfb 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -9,10 +9,8 @@ import asyncio from pathlib import Path import sys -from time import sleep import connexion -import threading sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.utils.common import initialize_environment From 866c5bcb15b1e51af743272892e38e9e3795d5b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 15:10:23 +0800 Subject: [PATCH 048/378] fixbug: merge bug --- metagpt/provider/openai_api.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 0f7100db8..88343373f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -25,26 +25,6 @@ from metagpt.utils.token_counter import ( ) -<<<<<<< HEAD -def retry(max_retries): - def decorator(f): - @wraps(f) - async def wrapper(*args, **kwargs): - for i in range(max_retries): - try: - return await f(*args, **kwargs) - except Exception as e: - error_str = traceback.format_exc() - logger.warning(f"Exception occurred: {str(e)}, stack:{error_str}. Retrying...") - if i == max_retries - 1: - raise - await asyncio.sleep(2 ** i) - return wrapper - return decorator - - -======= ->>>>>>> main class RateLimiter: """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" From 321f4a5a17bae23be5360f2ba7d2fadb76e66f9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 15:11:35 +0800 Subject: [PATCH 049/378] fixbug: merge bug --- metagpt/provider/openai_api.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 0f7100db8..88343373f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -25,26 +25,6 @@ from metagpt.utils.token_counter import ( ) -<<<<<<< HEAD -def retry(max_retries): - def decorator(f): - @wraps(f) - async def wrapper(*args, **kwargs): - for i in range(max_retries): - try: - return await f(*args, **kwargs) - except Exception as e: - error_str = traceback.format_exc() - logger.warning(f"Exception occurred: {str(e)}, stack:{error_str}. Retrying...") - if i == max_retries - 1: - raise - await asyncio.sleep(2 ** i) - return wrapper - return decorator - - -======= ->>>>>>> main class RateLimiter: """Rate control class, each call goes through wait_if_needed, sleep if rate control is needed""" From 341037601a89af510e9efdb598168562c4a278d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 19:01:07 +0800 Subject: [PATCH 050/378] feat: + unit test --- metagpt/learn/text_to_embedding.py | 23 +++++++++++ metagpt/learn/text_to_image.py | 23 +++++++++++ metagpt/learn/text_to_speech.py | 29 +++++++++++++ metagpt/tools/azure_tts.py | 2 +- tests/metagpt/learn/__init__.py | 0 tests/metagpt/learn/test_text_to_embedding.py | 40 ++++++++++++++++++ tests/metagpt/learn/test_text_to_image.py | 41 +++++++++++++++++++ tests/metagpt/learn/test_text_to_speech.py | 40 ++++++++++++++++++ 8 files changed, 197 insertions(+), 1 deletion(-) create mode 100644 metagpt/learn/text_to_embedding.py create mode 100644 metagpt/learn/text_to_image.py create mode 100644 metagpt/learn/text_to_speech.py create mode 100644 tests/metagpt/learn/__init__.py create mode 100644 tests/metagpt/learn/test_text_to_embedding.py create mode 100644 tests/metagpt/learn/test_text_to_image.py create mode 100644 tests/metagpt/learn/test_text_to_speech.py diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py new file mode 100644 index 000000000..b1395a61a --- /dev/null +++ b/metagpt/learn/text_to_embedding.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : text_to_embedding.py +@Desc : Text-to-Embedding skill, which provides text-to-embedding functionality. +""" + +from metagpt.tools.openai_text_2_embedding import oas3_openai_text_2_embedding +from metagpt.utils.common import initialize_environment + + +def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): + """Text to embedding + + :param text: The text used for embedding. + :param model: One of ['text-embedding-ada-002'], ID of the model to use. For more details, checkout: `https://api.openai.com/v1/models`. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. + """ + initialize_environment() + return oas3_openai_text_2_embedding(text, model=model, openai_api_key=openai_api_key) \ No newline at end of file diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py new file mode 100644 index 000000000..87668a13f --- /dev/null +++ b/metagpt/learn/text_to_image.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : text_to_image.py +@Desc : Text-to-Image skill, which provides text-to-image functionality. +""" + +from metagpt.tools.openai_text_2_image import oas3_openai_text_2_image +from metagpt.utils.common import initialize_environment + + +def text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): + """Text to image + + :param text: The text used for image conversion. + :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` + :param size_type: One of ['256x256', '512x512', '1024x1024'] + :return: The image data is returned in Base64 encoding. + """ + initialize_environment() + return oas3_openai_text_2_image(text, size_type, openai_api_key) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py new file mode 100644 index 000000000..909a9dca1 --- /dev/null +++ b/metagpt/learn/text_to_speech.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : text_to_speech.py +@Desc : Text-to-Speech skill, which provides text-to-speech functionality +""" + +from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.utils.common import initialize_environment + + +def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region=""): + """Text to speech + For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + + :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param voice: For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts`, `https://speech.microsoft.com/portal/voicegallery` + :param style: Speaking style to express different emotions like cheerfulness, empathy, and calm. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param role: With roles, the same voice can act as a different age and gender. For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` + :param text: The text used for voice conversion. + :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` + :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. + :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. + + """ + initialize_environment() + return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 2ec1539ef..21e8f1b6c 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -62,7 +62,7 @@ class AzureTTS: # Export def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): - """oas3/tts/azsure + """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` :param lang: The value can contain a language code such as en (English), or a locale such as en-US (English - United States). For more details, checkout: `https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` diff --git a/tests/metagpt/learn/__init__.py b/tests/metagpt/learn/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/metagpt/learn/test_text_to_embedding.py b/tests/metagpt/learn/test_text_to_embedding.py new file mode 100644 index 000000000..c85e5dde8 --- /dev/null +++ b/tests/metagpt/learn/test_text_to_embedding.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : test_text_to_embedding.py +@Desc : Unit tests. +""" + +import asyncio +import base64 + +from pydantic import BaseModel + +from metagpt.learn.text_to_embedding import text_to_embedding + + +async def mock_text_to_embedding(): + class Input(BaseModel): + input: str + + inputs = [ + {"input": "Panda emoji"} + ] + + for i in inputs: + seed = Input(**i) + data = text_to_embedding(seed.input) + v = ResultEmbedding(**data) + assert len(v.data) > 0 + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_text_to_embedding()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py new file mode 100644 index 000000000..bfcb1db25 --- /dev/null +++ b/tests/metagpt/learn/test_text_to_image.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : test_text_to_image.py +@Desc : Unit tests. +""" +import asyncio +import base64 + +from pydantic import BaseModel + +from metagpt.learn.text_to_image import text_to_image + + +async def mock_text_to_image(): + class Input(BaseModel): + input: str + size_type: str + + inputs = [ + {"input": "Panda emoji", "size_type": "256x256"} + ] + + for i in inputs: + seed = Input(**i) + base64_data = text_to_image(seed.input) + assert base64_data != "" + print(f"{seed.input} -> {base64_data}") + assert base64.b64decode(base64_data, validate=True) + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_text_to_image()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() diff --git a/tests/metagpt/learn/test_text_to_speech.py b/tests/metagpt/learn/test_text_to_speech.py new file mode 100644 index 000000000..dbb599e38 --- /dev/null +++ b/tests/metagpt/learn/test_text_to_speech.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : test_text_to_speech.py +@Desc : Unit tests. +""" +import asyncio +import base64 + +from pydantic import BaseModel + +from metagpt.learn.text_to_speech import text_to_speech + + +async def mock_text_to_speech(): + class Input(BaseModel): + input: str + + inputs = [ + {"input": "Panda emoji"} + ] + + for i in inputs: + seed = Input(**i) + base64_data = text_to_speech(seed.input) + assert base64_data != "" + print(f"{seed.input} -> {base64_data}") + assert base64.b64decode(base64_data, validate=True) + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_text_to_speech()) + loop.run_until_complete(task) + + +if __name__ == '__main__': + test_suite() \ No newline at end of file From 4f8187b6719689783352653fb9e0b5ef9eb55ac1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 19:29:51 +0800 Subject: [PATCH 051/378] feat: + METAGPT_TEXT_TO_IMAGE_MODEL --- config/config.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/config/config.yaml b/config/config.yaml index 303f4824b..6e9a61931 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -70,3 +70,6 @@ SD_T2I_API: "/sdapi/v1/txt2img" ### for Research MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k + +### Meta Models +#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL \ No newline at end of file From 99c143e8f301f89738eccdb4988552fc0a4a8cec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 20:09:06 +0800 Subject: [PATCH 052/378] feat: +metagpt text to image --- .gitignore | 1 + .well-known/metagpt_oas3_api.yaml | 47 +++++++- metagpt/tools/metagpt_text_to_image.py | 112 ++++++++++++++++++ ...bedding.py => openai_text_to_embedding.py} | 6 +- ...ext_2_image.py => openai_text_to_image.py} | 6 +- 5 files changed, 164 insertions(+), 8 deletions(-) create mode 100644 metagpt/tools/metagpt_text_to_image.py rename metagpt/tools/{openai_text_2_embedding.py => openai_text_to_embedding.py} (94%) rename metagpt/tools/{openai_text_2_image.py => openai_text_to_image.py} (94%) diff --git a/.gitignore b/.gitignore index c4c79c733..2cba27484 100644 --- a/.gitignore +++ b/.gitignore @@ -163,3 +163,4 @@ workspace/* *.mmd tmp output.wav +tmp.png diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 7ae10579c..a226181a5 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -71,7 +71,7 @@ paths: /txt2img/openai: post: summary: "Convert Text to Base64-encoded Image Data Stream" - operationId: openai_text_2_image.oas3_openai_text_2_image + operationId: openai_text_to_image.oas3_openai_text_to_image requestBody: required: true content: @@ -109,7 +109,7 @@ paths: /txt2embedding/openai: post: summary: Text to embedding - operationId: openai_text_2_embedding.oas3_openai_text_2_embedding + operationId: openai_text_to_embedding.oas3_openai_text_to_embedding description: Retrieve an embedding for the provided text using the OpenAI API. requestBody: content: @@ -144,6 +144,49 @@ paths: application/json: schema: $ref: "#/components/schemas/Error" + + /txt2image/metagpt: + post: + summary: "Text to Image" + description: "Generate an image from the provided text using the MetaGPT Text-to-Image API." + operationId: metagpt_text_to_image.oas3_metagpt_text_to_image + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - text + properties: + text: + type: string + description: "The text used for image conversion." + size_type: + type: string + enum: ["512x512", "512x768"] + default: "512x512" + description: "Size of the generated image." + model_url: + type: string + description: "Model reset API URL for text-to-image." + default: "" + responses: + '200': + description: "Base64-encoded image data." + content: + application/json: + schema: + type: object + properties: + image_data: + type: string + format: base64 + '400': + description: "Bad Request" + '500': + description: "Internal Server Error" + components: schemas: Embedding: diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py new file mode 100644 index 000000000..393215df0 --- /dev/null +++ b/metagpt/tools/metagpt_text_to_image.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : metagpt_text_to_image.py +@Desc : MetaGPT Text-to-Image OAS3 api, which provides text-to-image functionality. +""" +import base64 +import os +import sys +from pathlib import Path +from typing import List, Dict + +import requests +from pydantic import BaseModel + +sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' +from metagpt.utils.common import initialize_environment +from metagpt.logs import logger + + +class MetaGPTText2Image: + def __init__(self, model_url): + """ + :param model_url: Model reset api url + """ + self.model_url = model_url if model_url else os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + + def text_2_image(self, text, size_type="512x512"): + """Text to image + + :param text: The text used for image conversion. + :param size_type: One of ['512x512', '512x768'] + :return: The image data is returned in Base64 encoding. + """ + + headers = { + "Content-Type": "application/json" + } + dims = size_type.split("x") + data = { + "prompt": text, + "negative_prompt": "(easynegative:0.8),black, dark,Low resolution", + "override_settings": {"sd_model_checkpoint": "galaxytimemachinesGTM_photoV20"}, + "seed": -1, + "batch_size": 1, + "n_iter": 1, + "steps": 20, + "cfg_scale": 11, + "width": int(dims[0]), + "height": int(dims[1]), # 768, + "restore_faces": False, + "tiling": False, + "do_not_save_samples": False, + "do_not_save_grid": False, + "enable_hr": False, + "hr_scale": 2, + "hr_upscaler": "Latent", + "hr_second_pass_steps": 0, + "hr_resize_x": 0, + "hr_resize_y": 0, + "hr_upscale_to_x": 0, + "hr_upscale_to_y": 0, + "truncate_x": 0, + "truncate_y": 0, + "applied_old_hires_behavior_to": None, + "eta": None, + "sampler_index": "DPM++ SDE Karras", + "alwayson_scripts": {}, + } + + class ImageResult(BaseModel): + images: List + parameters: Dict + + try: + response = requests.post(self.model_url, headers=headers, json=data) + response.raise_for_status() # Raise an exception for 4xx or 5xx responses + result = ImageResult(**response.json()) + if len(result.images) == 0: + return "" + return result.images[0] + except requests.exceptions.RequestException as e: + logger.error(f"An error occurred:{e}") + return "" + + +# Export +def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): + """Text to image + + :param text: The text used for image conversion. + :param model_url: Model reset api + :param size_type: One of ['512x512', '512x768'] + :return: The image data is returned in Base64 encoding. + """ + if not text: + return "" + if not model_url: + model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + return MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) + + +if __name__ == "__main__": + initialize_environment() + + v = oas3_metagpt_text_2_image("Panda emoji") + data = base64.b64decode(v) + with open("tmp.png", mode="wb") as writer: + writer.write(data) + print(v) diff --git a/metagpt/tools/openai_text_2_embedding.py b/metagpt/tools/openai_text_to_embedding.py similarity index 94% rename from metagpt/tools/openai_text_2_embedding.py rename to metagpt/tools/openai_text_to_embedding.py index eb90a1ea9..9eddd5bc1 100644 --- a/metagpt/tools/openai_text_2_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -3,7 +3,7 @@ """ @Time : 2023/8/18 @Author : mashenquan -@File : openai_text_2_embedding.py +@File : openai_text_to_embedding.py @Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality. For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ @@ -70,7 +70,7 @@ class OpenAIText2Embedding: # Export -def oas3_openai_text_2_embedding(text, model="text-embedding-ada-002", openai_api_key=""): +def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): """Text to embedding :param text: The text used for embedding. @@ -88,5 +88,5 @@ def oas3_openai_text_2_embedding(text, model="text-embedding-ada-002", openai_ap if __name__ == "__main__": initialize_environment() - v = oas3_openai_text_2_embedding("Panda emoji") + v = oas3_openai_text_to_embedding("Panda emoji") print(v) diff --git a/metagpt/tools/openai_text_2_image.py b/metagpt/tools/openai_text_to_image.py similarity index 94% rename from metagpt/tools/openai_text_2_image.py rename to metagpt/tools/openai_text_to_image.py index 50c007626..6ec96d166 100644 --- a/metagpt/tools/openai_text_2_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -3,7 +3,7 @@ """ @Time : 2023/8/17 @Author : mashenquan -@File : openai_text_2_image.py +@File : openai_text_to_image.py @Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality. """ import base64 @@ -78,7 +78,7 @@ class OpenAIText2Image: # Export -def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key=""): +def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): """Text to image :param text: The text used for image conversion. @@ -96,5 +96,5 @@ def oas3_openai_text_2_image(text, size_type: str = "1024x1024", openai_api_key= if __name__ == "__main__": initialize_environment() - v = oas3_openai_text_2_image("Panda emoji") + v = oas3_openai_text_to_image("Panda emoji") print(v) From 3715a69e3f3df119477fd9f20e1d526afd94c115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 20:22:52 +0800 Subject: [PATCH 053/378] feat: update text_to_image skill --- metagpt/learn/text_to_embedding.py | 7 +++++-- metagpt/learn/text_to_image.py | 15 +++++++++++---- metagpt/learn/text_to_speech.py | 7 ++++++- tests/metagpt/learn/test_text_to_image.py | 2 +- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index b1395a61a..281815ca6 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -6,8 +6,9 @@ @File : text_to_embedding.py @Desc : Text-to-Embedding skill, which provides text-to-embedding functionality. """ +import os -from metagpt.tools.openai_text_2_embedding import oas3_openai_text_2_embedding +from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding from metagpt.utils.common import initialize_environment @@ -20,4 +21,6 @@ def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ initialize_environment() - return oas3_openai_text_2_embedding(text, model=model, openai_api_key=openai_api_key) \ No newline at end of file + if os.environ.get("OPENAI_API_KEY") or openai_api_key: + return oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) + raise EnvironmentError diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 87668a13f..0932dfe07 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,18 +6,25 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ +import os -from metagpt.tools.openai_text_2_image import oas3_openai_text_2_image +from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image +from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image from metagpt.utils.common import initialize_environment -def text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): +def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url=""): """Text to image :param text: The text used for image conversion. :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` - :param size_type: One of ['256x256', '512x512', '1024x1024'] + :param size_type: If using OPENAI, the available size options are ['256x256', '512x512', '1024x1024'], while for MetaGPT, the options are ['512x512', '512x768']. + :param model_url: MetaGPT model url :return: The image data is returned in Base64 encoding. """ initialize_environment() - return oas3_openai_text_2_image(text, size_type, openai_api_key) + if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL") or model_url: + return oas3_metagpt_text_to_image(text, size_type, model_url) + if os.environ.get("OPENAI_API_KEY") or openai_api_key: + return oas3_openai_text_to_image(text, size_type, openai_api_key) + raise EnvironmentError diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 909a9dca1..b89b5a9c4 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -6,6 +6,7 @@ @File : text_to_speech.py @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ +import os from metagpt.tools.azure_tts import oas3_azsure_tts from metagpt.utils.common import initialize_environment @@ -26,4 +27,8 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect """ initialize_environment() - return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ + (subscription_key and region): + return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + + raise EnvironmentError diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py index bfcb1db25..545c8a3ef 100644 --- a/tests/metagpt/learn/test_text_to_image.py +++ b/tests/metagpt/learn/test_text_to_image.py @@ -20,7 +20,7 @@ async def mock_text_to_image(): size_type: str inputs = [ - {"input": "Panda emoji", "size_type": "256x256"} + {"input": "Panda emoji", "size_type": "512x512"} ] for i in inputs: From df5a50f6e677fda08605fcbb44d7048642e76fc0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 18 Aug 2023 20:23:33 +0800 Subject: [PATCH 054/378] feat: update text_to_image skill --- metagpt/learn/text_to_speech.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index b89b5a9c4..1b81097b8 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -12,7 +12,8 @@ from metagpt.tools.azure_tts import oas3_azsure_tts from metagpt.utils.common import initialize_environment -def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region=""): +def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", + subscription_key="", region=""): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -28,7 +29,7 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect """ initialize_environment() if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ - (subscription_key and region): + (subscription_key and region): return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) raise EnvironmentError From f31b60309ad56faa4acb363f38f5b4dbd55a22c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 19 Aug 2023 21:57:09 +0800 Subject: [PATCH 055/378] feat: Config isolation at the object level. --- metagpt/config.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 21f180455..ac969f2f9 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,7 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -提供配置，单例 +@Desc: Provide configuration, singleton. +@Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config """ import os @@ -28,10 +29,13 @@ class NotConfiguredException(Exception): class Config(metaclass=Singleton): """ - 常规使用方法： + For example: + + ```python config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) + ``` """ _instance = None @@ -41,12 +45,13 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) + logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -85,20 +90,27 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") + # Update environment variables + for k, v in self._configs.items(): + os.environ[k] = str(v) + for attribute, value in vars(self).items(): + if attribute == "_configs": + continue + os.environ[attribute] = str(value) + def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" + """Load in decreasing priority from `config/key.yaml`, `config/config.yaml`, and environment variables.""" configs.update(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # 加载本地 YAML 文件 + # Load local YAML file. with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: continue - os.environ.update({k: v for k, v in yaml_data.items() if isinstance(v, str)}) configs.update(yaml_data) def _get(self, *args, **kwargs): @@ -111,5 +123,3 @@ class Config(metaclass=Singleton): raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value - -CONFIG = Config() From 291af5ad01bef9f6dbaa29305a3b13b29e21763b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 10:19:43 +0800 Subject: [PATCH 056/378] feat: + Config.options --- metagpt/config.py | 25 ++++++++++++++++--------- tests/metagpt/utils/test_config.py | 13 +++++++++++++ 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index ac969f2f9..6f3f9732a 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -3,6 +3,8 @@ """ @Desc: Provide configuration, singleton. @Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config +@Desc: `os.environ` doesn't support personalization, while `Config` does. + Hence, the parameter reading priority is `Config` first, and if not found, then `os.environ`. """ import os @@ -90,14 +92,6 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") - # Update environment variables - for k, v in self._configs.items(): - os.environ[k] = str(v) - for attribute, value in vars(self).items(): - if attribute == "_configs": - continue - os.environ[attribute] = str(value) - def _init_with_config_files_and_env(self, configs: dict, yaml_file): """Load in decreasing priority from `config/key.yaml`, `config/config.yaml`, and environment variables.""" configs.update(os.environ) @@ -117,9 +111,22 @@ class Config(metaclass=Singleton): return self._configs.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """从config/key.yaml / config/config.yaml / env三处找值，找不到报错""" + """Retrieve value from `config/key.yaml`, `config/config.yaml`, and environment variables. + Raise an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value + @property + def options(self): + """Return key-value configuration parameters.""" + opts = {} + for k, v in self._configs.items(): + opts[k] = v + for attribute, value in vars(self).items(): + if attribute == "_configs": + continue + opts[attribute] = value + return opts + diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index 558a4e5a4..475bac22b 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -4,7 +4,9 @@ @Time : 2023/5/1 11:19 @Author : alexanderwu @File : test_config.py +@Modified By: mashenquan, 2013/8/20, add `test_options` """ +from pathlib import Path import pytest @@ -29,3 +31,14 @@ def test_config_yaml_file_not_exists(): with pytest.raises(Exception) as exc_info: config.get('OPENAI_BASE_URL') assert str(exc_info.value) == "Key 'OPENAI_BASE_URL' not found in environment variables or in the YAML file" + + +def test_options(): + filename = Path(__file__).resolve().parent.parent.parent.parent / "config/config.yaml" + config = Config(filename) + opts = config.options + assert opts + + +if __name__ == '__main__': + test_options() From d764b8e6fa3fbbdcfc6f289b0f4495b6c7289d61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 10:26:26 +0800 Subject: [PATCH 057/378] feat: Remove global configuration CONFIG --- metagpt/config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 6f3f9732a..6e2cf0a3f 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -29,7 +29,7 @@ class NotConfiguredException(Exception): super().__init__(self.message) -class Config(metaclass=Singleton): +class Config: """ For example: @@ -40,7 +40,6 @@ class Config(metaclass=Singleton): ``` """ - _instance = None key_yaml_file = PROJECT_ROOT / "config/key.yaml" default_yaml_file = PROJECT_ROOT / "config/config.yaml" From f45a8e52842ca2b03f936132b3c51afaeeb2e9a6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 17:33:13 +0800 Subject: [PATCH 058/378] feat: Remove global configuration , enable configuration support for business isolation. --- metagpt/actions/action.py | 8 +- metagpt/actions/analyze_dep_libs.py | 5 +- metagpt/actions/debug_error.py | 5 +- metagpt/actions/design_api.py | 11 +-- metagpt/actions/design_api_review.py | 5 +- metagpt/actions/design_filenames.py | 5 +- metagpt/actions/project_management.py | 5 +- metagpt/actions/research.py | 24 ++++-- metagpt/actions/run_code.py | 5 +- metagpt/actions/search_and_summarize.py | 11 ++- metagpt/actions/write_code.py | 5 +- metagpt/actions/write_code_review.py | 5 +- metagpt/actions/write_prd.py | 7 +- metagpt/actions/write_prd_review.py | 5 +- metagpt/actions/write_test.py | 5 +- metagpt/config.py | 4 +- metagpt/document_store/faiss_store.py | 8 +- metagpt/llm.py | 20 ----- metagpt/management/skill_manager.py | 3 +- metagpt/manager.py | 5 +- metagpt/memory/longterm_memory.py | 9 +- metagpt/memory/memory_storage.py | 9 +- metagpt/provider/anthropic_api.py | 15 +++- metagpt/provider/openai_api.py | 82 +++++++++++++------ metagpt/roles/architect.py | 6 +- metagpt/roles/engineer.py | 6 +- metagpt/roles/product_manager.py | 6 +- metagpt/roles/project_manager.py | 6 +- metagpt/roles/qa_engineer.py | 4 +- metagpt/roles/role.py | 30 ++++--- metagpt/software_company.py | 25 +++++- metagpt/tools/search_engine.py | 38 +++++---- metagpt/tools/search_engine_ddg.py | 48 +++++------ metagpt/tools/search_engine_googleapi.py | 13 +-- metagpt/tools/search_engine_serpapi.py | 6 +- metagpt/tools/search_engine_serper.py | 4 +- metagpt/tools/web_browser_engine.py | 26 ++++-- .../tools/web_browser_engine_playwright.py | 24 ++++-- metagpt/tools/web_browser_engine_selenium.py | 19 +++-- metagpt/utils/mermaid.py | 22 +++-- startup.py | 16 ++-- tests/metagpt/actions/test_write_code.py | 14 +++- tests/metagpt/memory/test_longterm_memory.py | 21 +++-- tests/metagpt/test_environment.py | 41 +++++++--- tests/metagpt/test_llm.py | 7 +- tests/metagpt/tools/test_search_engine.py | 9 +- .../metagpt/tools/test_web_browser_engine.py | 8 +- .../test_web_browser_engine_playwright.py | 20 +++-- .../tools/test_web_browser_engine_selenium.py | 15 ++-- tests/metagpt/utils/test_config.py | 15 +--- 50 files changed, 437 insertions(+), 278 deletions(-) delete mode 100644 metagpt/llm.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index fa0d592a3..899c2515c 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : action.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from abc import ABC from typing import Optional @@ -11,15 +12,14 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.llm import LLM from metagpt.utils.common import OutputParser from metagpt.logs import logger + class Action(ABC): - def __init__(self, name: str = '', context=None, llm: LLM = None): + def __init__(self, options, name: str = '', context=None, llm=None): + self.options = options self.name: str = name - if llm is None: - llm = LLM() self.llm = llm self.context = context self.prefix = "" diff --git a/metagpt/actions/analyze_dep_libs.py b/metagpt/actions/analyze_dep_libs.py index 23c35cdf8..d7b251ead 100644 --- a/metagpt/actions/analyze_dep_libs.py +++ b/metagpt/actions/analyze_dep_libs.py @@ -4,6 +4,7 @@ @Time : 2023/5/19 12:01 @Author : alexanderwu @File : analyze_dep_libs.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action @@ -26,8 +27,8 @@ Focus only on the names of shared dependencies, do not add any other explanation class AnalyzeDepLibs(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.desc = "根据上下文，分析程序运行依赖库" async def run(self, requirement, filepaths_string): diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index d69a22dba..78c970337 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import re @@ -25,8 +26,8 @@ Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. """ class DebugError(Action): - def __init__(self, name="DebugError", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="DebugError", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) # async def run(self, code, error): # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index 1447eacc3..eb08cb9f0 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:26 @Author : alexanderwu @File : design_api.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import shutil from pathlib import Path @@ -90,8 +91,8 @@ OUTPUT_MAPPING = { class WriteDesign(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ "data structures, library tables, processes, and paths. Please provide your design, feedback " \ "clearly and in detail." @@ -106,15 +107,15 @@ class WriteDesign(Action): def _save_prd(self, docs_path, resources_path, prd): prd_file = docs_path / 'prd.md' quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') + mermaid_to_file(options=self.options, mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / 'competitive_analysis') logger.info(f"Saving PRD to {prd_file}") prd_file.write_text(prd) def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(data_api_design, resources_path / 'data_api_design') - mermaid_to_file(seq_flow, resources_path / 'seq_flow') + mermaid_to_file(options=self.options, mermaid_code=data_api_design, output_file_without_suffix=resources_path / 'data_api_design') + mermaid_to_file(options=self.options, mermaid_code=seq_flow, output_file_without_suffix=resources_path / 'seq_flow') system_design_file = docs_path / 'system_design.md' logger.info(f"Saving System Designs to {system_design_file}") system_design_file.write_text(content) diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index 687a33652..ca4147cca 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -4,13 +4,14 @@ @Time : 2023/5/11 19:31 @Author : alexanderwu @File : design_api_review.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class DesignReview(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) async def run(self, prd, api_design): prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \ diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 6c3d8e803..1f71e9530 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -4,6 +4,7 @@ @Time : 2023/5/19 11:50 @Author : alexanderwu @File : design_filenames.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action from metagpt.logs import logger @@ -15,8 +16,8 @@ Do not add any other explanations, just return a Python string list.""" class DesignFilenames(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \ "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 89c59dcda..3d8aa9322 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 19:12 @Author : alexanderwu @File : project_management.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -103,8 +104,8 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, name="CreateTasks", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="CreateTasks", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) def _save(self, context, rsp): ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 81eb876dd..22b0eaa1d 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -1,5 +1,9 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + from __future__ import annotations import asyncio @@ -9,7 +13,6 @@ from typing import Callable from pydantic import parse_obj_as from metagpt.actions import Action -from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.tools.search_engine import SearchEngine from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType @@ -79,14 +82,15 @@ class CollectLinks(Action): """Action class to collect links from a search engine.""" def __init__( self, + options, name: str = "", *args, rank_func: Callable[[list[str]], None] | None = None, **kwargs, ): - super().__init__(name, *args, **kwargs) + super().__init__(options=options, name=name, *args, **kwargs) self.desc = "Collect links from a search engine." - self.search_engine = SearchEngine() + self.search_engine = SearchEngine(options=options) self.rank_func = rank_func async def run( @@ -126,7 +130,7 @@ class CollectLinks(Action): remove.pop() if len(remove) == 0: break - prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp) + prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, self.options.get("max_tokens_rsp")) logger.debug(prompt) queries = await self._aask(prompt, [system_text]) try: @@ -178,9 +182,10 @@ class WebBrowseAndSummarize(Action): **kwargs, ): super().__init__(*args, **kwargs) - if CONFIG.model_for_researcher_summary: - self.llm.model = CONFIG.model_for_researcher_summary + if self.options.get("model_for_researcher_summary"): + self.llm.model = self.options.get("model_for_researcher_summary") self.web_browser_engine = WebBrowserEngine( + options=self.options, engine=WebBrowserEngineType.CUSTOM if browse_func else None, run_func=browse_func, ) @@ -213,7 +218,8 @@ class WebBrowseAndSummarize(Action): for u, content in zip([url, *urls], contents): content = content.inner_text chunk_summaries = [] - for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp): + for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, + self.options.get("max_tokens_rsp")): logger.debug(prompt) summary = await self._aask(prompt, [system_text]) if summary == "Not relevant.": @@ -239,8 +245,8 @@ class ConductResearch(Action): """Action class to conduct research and generate a research report.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if CONFIG.model_for_researcher_report: - self.llm.model = CONFIG.model_for_researcher_report + if self.options.get("model_for_researcher_report"): + self.llm.model = self.options.get("model_for_researcher_report") async def run( self, diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index f69d2cd1a..824ed83fa 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import os import subprocess @@ -57,8 +58,8 @@ standard errors: {errs}; class RunCode(Action): - def __init__(self, name="RunCode", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="RunCode", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) @classmethod async def run_text(cls, code) -> Tuple[str, str]: diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 5e4cdaea0..80d1c52e4 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -4,11 +4,11 @@ @Time : 2023/5/23 17:26 @Author : alexanderwu @File : search_google.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pydantic from metagpt.actions import Action -from metagpt.config import Config from metagpt.logs import logger from metagpt.schema import Message from metagpt.tools.search_engine import SearchEngine @@ -101,17 +101,16 @@ You are a member of a professional butler team and will provide helpful suggesti class SearchAndSummarize(Action): - def __init__(self, name="", context=None, llm=None, engine=None, search_func=None): - self.config = Config() - self.engine = engine or self.config.search_engine + def __init__(self, options, name="", context=None, llm=None, engine=None, search_func=None): + self.engine = engine or options.get("search_engine") try: - self.search_engine = SearchEngine(self.engine, run_func=search_func) + self.search_engine = SearchEngine(options=options, engine=self.engine, run_func=search_func) except pydantic.ValidationError: self.search_engine = None self.result = "" - super().__init__(name, context, llm) + super().__init__(options=options, name=name, context=context, llm=llm) async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str: if self.search_engine is None: diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index cc122ef7a..9a2a2f81a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action @@ -43,8 +44,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): - def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="WriteCode", context: list[Message] = None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index 7f6a7a38e..d256c6bcb 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action @@ -62,8 +63,8 @@ FORMAT_EXAMPLE = """ class WriteCodeReview(Action): - def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="WriteCodeReview", context: list[Message] = None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0edd24d55..794d3ee9d 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -127,11 +128,11 @@ OUTPUT_MAPPING = { class WritePRD(Action): - def __init__(self, name="", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) async def run(self, requirements, *args, **kwargs) -> ActionOutput: - sas = SearchAndSummarize() + sas = SearchAndSummarize(options=self.options, llm=self.llm) # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 5ff9624c5..8c22f9c0a 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -4,13 +4,14 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd_review.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class WritePRDReview(Action): - def __init__(self, name, context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name, context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) self.prd = None self.desc = "Based on the PRD, conduct a PRD Review, providing clear and detailed feedback" self.prd_review_prompt_template = """ diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 5e50fdb55..94006005f 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -4,6 +4,7 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_test.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action from metagpt.utils.common import CodeParser @@ -30,8 +31,8 @@ you should correctly import the necessary classes based on these file locations! class WriteTest(Action): - def __init__(self, name="WriteTest", context=None, llm=None): - super().__init__(name, context, llm) + def __init__(self, options, name="WriteTest", context=None, llm=None): + super().__init__(options=options, name=name, context=context, llm=llm) async def write_code(self, prompt): code_rsp = await self._aask(prompt) diff --git a/metagpt/config.py b/metagpt/config.py index 6e2cf0a3f..076bc5eb7 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -118,8 +118,8 @@ class Config: return value @property - def options(self): - """Return key-value configuration parameters.""" + def runtime_options(self): + """Runtime key-value configuration parameters.""" opts = {} for k, v in self._configs.items(): opts[k] = v diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 051bc2507..d15eb4c21 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -4,6 +4,7 @@ @Time : 2023/5/25 10:20 @Author : alexanderwu @File : faiss_store.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pickle from pathlib import Path @@ -36,8 +37,11 @@ class FaissStore(LocalStore): store.index = index return store - def _write(self, docs, metadatas): - store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas) + def _write(self, docs, metadatas, **kwargs): + store = FAISS.from_texts(docs, + OpenAIEmbeddings(openai_api_version="2020-11-07", + openai_api_key=kwargs.get("OPENAI_API_KEY")), + metadatas=metadatas) return store def persist(self): diff --git a/metagpt/llm.py b/metagpt/llm.py deleted file mode 100644 index 6a9a9132f..000000000 --- a/metagpt/llm.py +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/5/11 14:45 -@Author : alexanderwu -@File : llm.py -""" - -from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM - -DEFAULT_LLM = LLM() -CLAUDE_LLM = Claude() - - -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/management/skill_manager.py b/metagpt/management/skill_manager.py index f067e6df6..4f141832a 100644 --- a/metagpt/management/skill_manager.py +++ b/metagpt/management/skill_manager.py @@ -4,11 +4,11 @@ @Time : 2023/6/5 01:44 @Author : alexanderwu @File : skill_manager.py +@Modified By: mashenquan, 2023/8/20. Remove useless `_llm` """ from metagpt.actions import Action from metagpt.const import PROMPT_PATH from metagpt.document_store.chromadb_store import ChromaStore -from metagpt.llm import LLM from metagpt.logs import logger Skill = Action @@ -18,7 +18,6 @@ class SkillManager: """用来管理所有技能""" def __init__(self): - self._llm = LLM() self._store = ChromaStore('skill_manager') self._skills: dict[str: Skill] = {} diff --git a/metagpt/manager.py b/metagpt/manager.py index 9d238c621..c4565808e 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -4,14 +4,15 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : manager.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -from metagpt.llm import LLM + from metagpt.logs import logger from metagpt.schema import Message class Manager: - def __init__(self, llm: LLM = LLM()): + def __init__(self, llm): self.llm = llm # Large Language Model self.role_directions = { "BOSS": "Product Manager", diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 3c2963613..041d335ac 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the implement of Long-term memory +""" +@Desc : the implement of Long-term memory +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from metagpt.logs import logger from metagpt.memory import Memory @@ -34,13 +37,13 @@ class LongTermMemory(Memory): self.add_batch(messages) self.msg_from_recover = False - def add(self, message: Message): + def add(self, message: Message, **kwargs): super(LongTermMemory, self).add(message) for action in self.rc.watch: if message.cause_by == action and not self.msg_from_recover: # currently, only add role's watching messages to its memory_storage # and ignore adding messages from recover repeatedly - self.memory_storage.add(message) + self.memory_storage.add(message, **kwargs) def remember(self, observed: list[Message], k=0) -> list[Message]: """ diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 5421e9e65..09cd67410 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -1,6 +1,9 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : the implement of memory storage +""" +@Desc : the implement of memory storage +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from typing import List from pathlib import Path @@ -61,13 +64,13 @@ class MemoryStorage(FaissStore): super(MemoryStorage, self).persist() logger.debug(f'Agent {self.role_id} persist memory into local') - def add(self, message: Message) -> bool: + def add(self, message: Message, **kwargs) -> bool: """ add message into memory storage""" docs = [message.content] metadatas = [{"message_ser": serialize_message(message)}] if not self.store: # init Faiss - self.store = self._write(docs, metadatas) + self.store = self._write(docs, metadatas, **kwargs) self._initialized = True else: self.store.add_texts(texts=docs, metadatas=metadatas) diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 03802a716..326d23a5c 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -4,17 +4,22 @@ @Time : 2023/7/21 11:15 @Author : Leo Xiao @File : anthropic_api.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ import anthropic from anthropic import Anthropic -from metagpt.config import CONFIG +from metagpt.config import Config class Claude2: + def __init__(self, options=None): + self.options = options or Config().runtime_options + def ask(self, prompt): - client = Anthropic(api_key=CONFIG.claude_api_key) + client = Anthropic(api_key=self.claude_api_key) res = client.completions.create( model="claude-2", @@ -24,7 +29,7 @@ class Claude2: return res.completion async def aask(self, prompt): - client = Anthropic(api_key=CONFIG.claude_api_key) + client = Anthropic(api_key=self.claude_api_key) res = client.completions.create( model="claude-2", @@ -32,3 +37,7 @@ class Claude2: max_tokens_to_sample=1000, ) return res.completion + + @property + def claude_api_key(self): + return self.options.get("claude_api_key") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 79121c8de..2e951b36f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -3,6 +3,8 @@ @Time : 2023/5/5 23:08 @Author : alexanderwu @File : openai.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ import asyncio import time @@ -12,10 +14,8 @@ import openai from openai.error import APIConnectionError from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type -from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI -from metagpt.utils.singleton import Singleton from metagpt.utils.token_counter import ( TOKEN_COSTS, count_message_tokens, @@ -56,13 +56,13 @@ class Costs(NamedTuple): total_budget: float -class CostManager(metaclass=Singleton): +class CostManager: """计算使用接口的开销""" - def __init__(self): + def __init__(self, options): self.total_prompt_tokens = 0 self.total_completion_tokens = 0 - self.total_cost = 0 + self.options = options self.total_budget = 0 def update_cost(self, prompt_tokens, completion_tokens, model): @@ -79,10 +79,9 @@ class CostManager(metaclass=Singleton): cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]) / 1000 self.total_cost += cost logger.info( - f"Total running cost: ${self.total_cost:.3f} | Max budget: ${CONFIG.max_budget:.3f} | " + f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" ) - CONFIG.total_cost = self.total_cost def get_total_prompt_tokens(self): """ @@ -115,6 +114,18 @@ class CostManager(metaclass=Singleton): """获得所有开销""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) + @property + def total_cost(self): + return self.options.get("total_cost", 0) + + @total_cost.setter + def total_cost(self, v): + self.options["total_cost"] = v + + @property + def max_budget(self): + return self.options.get("max_budget", 0) + def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") @@ -130,22 +141,23 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self): - self.__init_openai(CONFIG) + def __init__(self, options, cost_manager): + self._options = options + self.__init_openai() self.llm = openai - self.model = CONFIG.openai_api_model + self.model = self.openai_api_model self.auto_max_tokens = False - self._cost_manager = CostManager() + self._cost_manager = cost_manager RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): - openai.api_key = config.openai_api_key - if config.openai_api_base: - openai.api_base = config.openai_api_base - if config.openai_api_type: - openai.api_type = config.openai_api_type - openai.api_version = config.openai_api_version - self.rpm = int(config.get("RPM", 10)) + def __init_openai(self): + openai.api_key = self.openai_api_key + if self.openai_api_base: + openai.api_base = self.openai_api_base + if self.openai_api_type: + openai.api_type = self.openai_api_type + openai.api_version = self.openai_api_version + self.rpm = int(self._options.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) @@ -168,9 +180,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return full_reply_content def _cons_kwargs(self, messages: list[dict]) -> dict: - if CONFIG.openai_api_type == "azure": + if self._options.get("openai_api_type") == "azure": kwargs = { - "deployment_id": CONFIG.deployment_id, + "deployment_id": self._options.get("deployment_id"), "messages": messages, "max_tokens": self.get_max_tokens(messages), "n": 1, @@ -225,7 +237,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} - if CONFIG.calc_usage: + if self._options.get("calc_usage"): try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) @@ -264,7 +276,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return results def _update_costs(self, usage: dict): - if CONFIG.calc_usage: + if self._options.get("calc_usage"): try: prompt_tokens = int(usage['prompt_tokens']) completion_tokens = int(usage['completion_tokens']) @@ -277,5 +289,25 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_max_tokens(self, messages: list[dict]): if not self.auto_max_tokens: - return CONFIG.max_tokens_rsp - return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) + return self._options.get("max_tokens_rsp") + return get_max_completion_tokens(messages, self.model, self._options.get("max_tokens_rsp")) + + @property + def openai_api_model(self): + return self._options.get("openai_api_model") + + @property + def openai_api_key(self): + return self._options.get("openai_api_key") + + @property + def openai_api_base(self): + return self._options.get("openai_api_base") + + @property + def openai_api_type(self): + return self._options.get("openai_api_type") + + @property + def openai_api_version(self): + return self._options.get("openai_api_version") diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 00b6cb2eb..5a498c50b 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -4,6 +4,8 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : architect.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WritePRD @@ -12,8 +14,8 @@ from metagpt.roles import Role class Architect(Role): """Architect: Listen to PRD, responsible for designing API, designing code files""" - def __init__(self, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", + def __init__(self, options, cost_manager, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", constraints="Try to specify good open source tools as much as possible"): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WriteDesign]) self._watch({WritePRD}) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 072e53998..9da2b5a09 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -47,10 +47,10 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", + def __init__(self, options, cost_manager, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", n_borg=1, use_code_review=False): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WriteCode]) self.use_code_review = use_code_review if self.use_code_review: @@ -131,7 +131,7 @@ class Engineer(Role): async def _act_sp(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: - code = await WriteCode().run( + code = await WriteCode(options=self.options, llm=self._llm).run( context=self._rc.history, filename=todo ) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index b42e9bb29..bb69c8dfd 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,14 +4,16 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from metagpt.actions import BossRequirement, WritePRD from metagpt.roles import Role class ProductManager(Role): - def __init__(self, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", + def __init__(self, options, cost_manager, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", constraints=""): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WritePRD]) self._watch([BossRequirement]) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index ff374de13..3e8b36550 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -4,14 +4,16 @@ @Time : 2023/5/11 15:04 @Author : alexanderwu @File : project_manager.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WriteTasks from metagpt.roles import Role class ProjectManager(Role): - def __init__(self, name="Eve", profile="Project Manager", + def __init__(self, options, cost_manager, name="Eve", profile="Project Manager", goal="Improve team efficiency and deliver with quality and quantity", constraints=""): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions([WriteTasks]) self._watch([WriteDesign]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 65bf2cc5b..ac5df0dbd 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -20,13 +20,15 @@ from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP class QaEngineer(Role): def __init__( self, + options, + cost_manager, name="Edward", profile="QaEngineer", goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", test_round_allowed=5, ): - super().__init__(name, profile, goal, constraints) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index d3750495f..3c72876a5 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,17 +4,16 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ from __future__ import annotations -from typing import Iterable, Type +from typing import Iterable, Type, Dict from pydantic import BaseModel, Field - -# from metagpt.environment import Environment -from metagpt.config import CONFIG +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM from metagpt.actions import Action, ActionOutput -from metagpt.llm import LLM from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory from metagpt.schema import Message @@ -71,12 +70,13 @@ class RoleContext(BaseModel): todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) + options: Dict class Config: arbitrary_types_allowed = True def check(self, role_id: str): - if hasattr(CONFIG, "long_term_memory") and CONFIG.long_term_memory: + if self.options.get("long_term_memory"): self.long_term_memory.recover_memory(role_id, self) self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation @@ -93,13 +93,15 @@ class RoleContext(BaseModel): class Role: """角色/代理""" - def __init__(self, name="", profile="", goal="", constraints="", desc=""): - self._llm = LLM() + def __init__(self, options, cost_manager, name="", profile="", goal="", constraints="", desc=""): + self._options = options if options else {} + self._cost_manager = cost_manager + self._llm = LLM(options=self._options, cost_manager=cost_manager) self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] self._role_id = str(self._setting) - self._rc = RoleContext() + self._rc = RoleContext(options=options) def _reset(self): self._states = [] @@ -109,7 +111,7 @@ class Role: self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action("") + i = action(options=self._options, name="", llm=self._llm) else: i = action i.set_prefix(self._get_prefix(), self.profile) @@ -137,6 +139,14 @@ class Role: """获取角色描述（职位）""" return self._setting.profile + @property + def options(self): + return self._options + + @options.setter + def options(self, opts): + self._options.update(opts) + def _get_prefix(self): """获取角色前缀""" if self._setting.desc: diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..3f6f484b4 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,16 +4,21 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. """ +from typing import Dict + from pydantic import BaseModel, Field from metagpt.actions import BossRequirement -from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger +from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException +from metagpt.config import Config class SoftwareCompany(BaseModel): @@ -24,6 +29,8 @@ class SoftwareCompany(BaseModel): environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") + options: Dict = Field(default=Config().runtime_options) + cost_manager: CostManager = Field(default=CostManager(Config().runtime_options)) class Config: arbitrary_types_allowed = True @@ -35,12 +42,12 @@ class SoftwareCompany(BaseModel): def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment - CONFIG.max_budget = investment + self.options["max_budget"] = investment logger.info(f'Investment: ${investment}.') def _check_balance(self): - if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + if self.total_cost > self.max_budget: + raise NoMoneyException(self.total_cost, f'Insufficient funds: {self.max_budget}') def start_project(self, idea): """Start a project from publishing boss requirement.""" @@ -59,3 +66,13 @@ class SoftwareCompany(BaseModel): self._check_balance() await self.environment.run() return self.environment.history + + @property + def max_budget(self): + return self.options.get("max_budget", 0) + + @property + def total_cost(self): + return self.options.get("total_cost", 0) + + diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index d28700054..c82ae6595 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -4,13 +4,13 @@ @Time : 2023/5/6 20:15 @Author : alexanderwu @File : search_engine.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import importlib -from typing import Callable, Coroutine, Literal, overload +from typing import Callable, Coroutine, Literal, overload, Dict -from metagpt.config import CONFIG from metagpt.tools import SearchEngineType @@ -25,24 +25,26 @@ class SearchEngine: run_func: The function to run the search. engine: The search engine type. """ + def __init__( - self, - engine: SearchEngineType | None = None, - run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None, + self, + options: Dict, + engine: SearchEngineType | None = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None ): - engine = engine or CONFIG.search_engine + engine = engine or options.get("search_engine") if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper().run + run_func = importlib.import_module(module).SerpAPIWrapper(**options).run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" - run_func = importlib.import_module(module).SerperWrapper().run + run_func = importlib.import_module(module).SerperWrapper(**options).run elif engine == SearchEngineType.DIRECT_GOOGLE: module = "metagpt.tools.search_engine_googleapi" - run_func = importlib.import_module(module).GoogleAPIWrapper().run + run_func = importlib.import_module(module).GoogleAPIWrapper(**options).run elif engine == SearchEngineType.DUCK_DUCK_GO: module = "metagpt.tools.search_engine_ddg" - run_func = importlib.import_module(module).DDGAPIWrapper().run + run_func = importlib.import_module(module).DDGAPIWrapper(**options).run elif engine == SearchEngineType.CUSTOM_ENGINE: pass # run_func = run_func else: @@ -52,19 +54,19 @@ class SearchEngine: @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, ) -> list[dict[str, str]]: ... diff --git a/metagpt/tools/search_engine_ddg.py b/metagpt/tools/search_engine_ddg.py index 57bc61b82..78562c77e 100644 --- a/metagpt/tools/search_engine_ddg.py +++ b/metagpt/tools/search_engine_ddg.py @@ -1,11 +1,14 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from __future__ import annotations import asyncio import json from concurrent import futures -from typing import Literal, overload +from typing import Literal, overload, Optional try: from duckduckgo_search import DDGS @@ -15,8 +18,6 @@ except ImportError: "You can install it by running the command: `pip install -e.[search-ddg]`" ) -from metagpt.config import CONFIG - class DDGAPIWrapper: """Wrapper around duckduckgo_search API. @@ -25,43 +26,44 @@ class DDGAPIWrapper: """ def __init__( - self, - *, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, + self, + *, + global_proxy: Optional[str] = None, + loop: asyncio.AbstractEventLoop | None = None, + executor: futures.Executor | None = None, ): kwargs = {} - if CONFIG.global_proxy: - kwargs["proxies"] = CONFIG.global_proxy + if global_proxy: + kwargs["proxies"] = global_proxy self.loop = loop self.executor = executor self.ddgs = DDGS(**kwargs) @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, + focus: list[str] | None = None, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, + focus: list[str] | None = None, ) -> list[dict[str, str]]: ... async def run( - self, - query: str, - max_results: int = 8, - as_string: bool = True, + self, + query: str, + max_results: int = 8, + as_string: bool = True, ) -> str | list[dict]: """Return the results of a Google search using the official Google API diff --git a/metagpt/tools/search_engine_googleapi.py b/metagpt/tools/search_engine_googleapi.py index b9faf2ced..b5aeb5875 100644 --- a/metagpt/tools/search_engine_googleapi.py +++ b/metagpt/tools/search_engine_googleapi.py @@ -1,5 +1,8 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from __future__ import annotations import asyncio @@ -11,7 +14,6 @@ from urllib.parse import urlparse import httplib2 from pydantic import BaseModel, validator -from metagpt.config import CONFIG from metagpt.logs import logger try: @@ -27,6 +29,7 @@ except ImportError: class GoogleAPIWrapper(BaseModel): google_api_key: Optional[str] = None google_cse_id: Optional[str] = None + global_proxy: Optional[str] = None loop: Optional[asyncio.AbstractEventLoop] = None executor: Optional[futures.Executor] = None @@ -36,7 +39,6 @@ class GoogleAPIWrapper(BaseModel): @validator("google_api_key", always=True) @classmethod def check_google_api_key(cls, val: str): - val = val or CONFIG.google_api_key if not val: raise ValueError( "To use, make sure you provide the google_api_key when constructing an object. Alternatively, " @@ -47,8 +49,7 @@ class GoogleAPIWrapper(BaseModel): @validator("google_cse_id", always=True) @classmethod - def check_google_cse_id(cls, val: str): - val = val or CONFIG.google_cse_id + def check_google_cse_id(cls, val): if not val: raise ValueError( "To use, make sure you provide the google_cse_id when constructing an object. Alternatively, " @@ -60,8 +61,8 @@ class GoogleAPIWrapper(BaseModel): @property def google_api_client(self): build_kwargs = {"developerKey": self.google_api_key} - if CONFIG.global_proxy: - parse_result = urlparse(CONFIG.global_proxy) + if self.global_proxy: + parse_result = urlparse(self.global_proxy) proxy_type = parse_result.scheme if proxy_type == "https": proxy_type = "http" diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 750184198..1b93a91e9 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -4,13 +4,14 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator -from metagpt.config import CONFIG +from metagpt.config import Config class SerpAPIWrapper(BaseModel): @@ -32,7 +33,6 @@ class SerpAPIWrapper(BaseModel): @validator("serpapi_api_key", always=True) @classmethod def check_serpapi_api_key(cls, val: str): - val = val or CONFIG.serpapi_api_key if not val: raise ValueError( "To use, make sure you provide the serpapi_api_key when constructing an object. Alternatively, " @@ -112,4 +112,4 @@ class SerpAPIWrapper(BaseModel): if __name__ == "__main__": import fire - fire.Fire(SerpAPIWrapper().run) + fire.Fire(SerpAPIWrapper(Config().runtime_options).run) diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 0eec2694b..849839f05 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -4,6 +4,7 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import json from typing import Any, Dict, Optional, Tuple @@ -11,8 +12,6 @@ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator -from metagpt.config import CONFIG - class SerperWrapper(BaseModel): search_engine: Any #: :meta private: @@ -26,7 +25,6 @@ class SerperWrapper(BaseModel): @validator("serper_api_key", always=True) @classmethod def check_serper_api_key(cls, val: str): - val = val or CONFIG.serper_api_key if not val: raise ValueError( "To use, make sure you provide the serper_api_key when constructing an object. Alternatively, " diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index 453d87f31..da208dbc9 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -1,29 +1,33 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" from __future__ import annotations import importlib -from typing import Any, Callable, Coroutine, Literal, overload +from typing import Any, Callable, Coroutine, Literal, overload, Dict -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.tools import WebBrowserEngineType from metagpt.utils.parse_html import WebPage class WebBrowserEngine: def __init__( - self, - engine: WebBrowserEngineType | None = None, - run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, + self, + options: Dict, + engine: WebBrowserEngineType | None = None, + run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, ): - engine = engine or CONFIG.web_browser_engine + engine = engine or options.get("web_browser_engine") if engine == WebBrowserEngineType.PLAYWRIGHT: module = "metagpt.tools.web_browser_engine_playwright" - run_func = importlib.import_module(module).PlaywrightWrapper().run + run_func = importlib.import_module(module).PlaywrightWrapper(options=options).run elif engine == WebBrowserEngineType.SELENIUM: module = "metagpt.tools.web_browser_engine_selenium" - run_func = importlib.import_module(module).SeleniumWrapper().run + run_func = importlib.import_module(module).SeleniumWrapper(options=options).run elif engine == WebBrowserEngineType.CUSTOM: run_func = run_func else: @@ -47,6 +51,10 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs): - return await WebBrowserEngine(WebBrowserEngineType(engine_type), **kwargs).run(url, *urls) + conf = Config() + return await WebBrowserEngine(options=conf.runtime_options, + engine=WebBrowserEngineType(engine_type), + **kwargs).run(url, *urls) + fire.Fire(main) diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index 030e7701b..199f8a0d1 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -1,14 +1,18 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + from __future__ import annotations import asyncio import sys from pathlib import Path -from typing import Literal +from typing import Literal, Dict from playwright.async_api import async_playwright -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.logs import logger from metagpt.utils.parse_html import WebPage @@ -24,18 +28,20 @@ class PlaywrightWrapper: def __init__( self, + options: Dict, browser_type: Literal["chromium", "firefox", "webkit"] | None = None, launch_kwargs: dict | None = None, **kwargs, ) -> None: + self.options = options if browser_type is None: - browser_type = CONFIG.playwright_browser_type + browser_type = options.get("playwright_browser_type") self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if CONFIG.global_proxy and "proxy" not in launch_kwargs: + if options.get("global_proxy") and "proxy" not in launch_kwargs: args = launch_kwargs.get("args", []) if not any(str.startswith(i, "--proxy-server=") for i in args): - launch_kwargs["proxy"] = {"server": CONFIG.global_proxy} + launch_kwargs["proxy"] = {"server": options.get("global_proxy")} self.launch_kwargs = launch_kwargs context_kwargs = {} if "ignore_https_errors" in kwargs: @@ -75,8 +81,8 @@ class PlaywrightWrapper: executable_path = Path(browser_type.executable_path) if not executable_path.exists() and "executable_path" not in self.launch_kwargs: kwargs = {} - if CONFIG.global_proxy: - kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy} + if self.options.get("global_proxy"): + kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")} await _install_browsers(self.browser_type, **kwargs) if self._has_run_precheck: @@ -144,6 +150,8 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs): - return await PlaywrightWrapper(browser_type, **kwargs).run(url, *urls) + return await PlaywrightWrapper(options=Config().runtime_options, + browser_type=browser_type, + **kwargs).run(url, *urls) fire.Fire(main) diff --git a/metagpt/tools/web_browser_engine_selenium.py b/metagpt/tools/web_browser_engine_selenium.py index d727709b8..b0fcb3fe1 100644 --- a/metagpt/tools/web_browser_engine_selenium.py +++ b/metagpt/tools/web_browser_engine_selenium.py @@ -1,17 +1,21 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + from __future__ import annotations import asyncio import importlib from concurrent import futures from copy import deepcopy -from typing import Literal +from typing import Literal, Dict from selenium.webdriver.common.by import By from selenium.webdriver.support import expected_conditions as EC from selenium.webdriver.support.wait import WebDriverWait -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.utils.parse_html import WebPage @@ -29,6 +33,7 @@ class SeleniumWrapper: def __init__( self, + options: Dict, browser_type: Literal["chrome", "firefox", "edge", "ie"] | None = None, launch_kwargs: dict | None = None, *, @@ -36,11 +41,11 @@ class SeleniumWrapper: executor: futures.Executor | None = None, ) -> None: if browser_type is None: - browser_type = CONFIG.selenium_browser_type + browser_type = options.get("selenium_browser_type") self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if CONFIG.global_proxy and "proxy-server" not in launch_kwargs: - launch_kwargs["proxy-server"] = CONFIG.global_proxy + if options.get("global_proxy") and "proxy-server" not in launch_kwargs: + launch_kwargs["proxy-server"] = options.get("global_proxy") self.executable_path = launch_kwargs.pop("executable_path", None) self.launch_args = [f"--{k}={v}" for k, v in launch_kwargs.items()] @@ -118,6 +123,8 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chrome", **kwargs): - return await SeleniumWrapper(browser_type, **kwargs).run(url, *urls) + return await SeleniumWrapper(options=Config().runtime_options, + browser_type=browser_type, + **kwargs).run(url, *urls) fire.Fire(main) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 24aabe8ae..1245671fb 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -4,19 +4,21 @@ @Time : 2023/7/4 10:53 @Author : alexanderwu @File : mermaid.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import subprocess from pathlib import Path -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.const import PROJECT_ROOT from metagpt.logs import logger from metagpt.utils.common import check_cmd_exists -def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: +def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """suffix: png/svg/pdf + :param options: runtime context options, created by `Config` class object and changed in flow pipeline :param mermaid_code: mermaid code :param output_file_without_suffix: output filename :param width: @@ -36,12 +38,12 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") - if CONFIG.puppeteer_config: + if options.get("puppeteer_config"): subprocess.run( [ - CONFIG.mmdc, + options.get("mmdc"), "-p", - CONFIG.puppeteer_config, + options.get("puppeteer_config"), "-i", str(tmp), "-o", @@ -53,7 +55,7 @@ def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height ] ) else: - subprocess.run([CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) + subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) return 0 @@ -109,6 +111,8 @@ MMC2 = """sequenceDiagram if __name__ == "__main__": - # logger.info(print_members(print_members)) - mermaid_to_file(MMC1, PROJECT_ROOT / "tmp/1.png") - mermaid_to_file(MMC2, PROJECT_ROOT / "tmp/2.png") + conf = Config() + mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1, + output_file_without_suffix=PROJECT_ROOT / "tmp/1.png") + mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2, + output_file_without_suffix=PROJECT_ROOT / "tmp/2.png") diff --git a/startup.py b/startup.py index f37b5286c..116e4073d 100644 --- a/startup.py +++ b/startup.py @@ -1,5 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; + Change cost control from global to company level. +""" + import asyncio import fire @@ -11,14 +16,15 @@ from metagpt.software_company import SoftwareCompany async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False): """Run a startup. Be a boss.""" + company = SoftwareCompany() - company.hire([ProductManager(), - Architect(), - ProjectManager(), - Engineer(n_borg=5, use_code_review=code_review)]) + company.hire([ProductManager(options=company.options, cost_manager=company.cost_manager), + Architect(options=company.options, cost_manager=company.cost_manager), + ProjectManager(options=company.options, cost_manager=company.cost_manager), + Engineer(n_borg=5, use_code_review=code_review, options=company.options, cost_manager=company.cost_manager)]) if run_tests: # developing features: run tests on the spot and identify bugs (bug fixing capability comes soon!) - company.hire([QaEngineer()]) + company.hire([QaEngineer(options=company.options, cost_manager=company.cost_manager)]) company.invest(investment) company.start_project(idea) await company.run(n_round=n_round) diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 7bb18ddf2..04216ad7c 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -4,11 +4,13 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : test_write_code.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pytest +from metagpt.config import Config +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager from metagpt.actions.write_code import WriteCode -from metagpt.llm import LLM from metagpt.logs import logger from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @@ -16,9 +18,12 @@ from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE @pytest.mark.asyncio async def test_write_code(): api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" - write_code = WriteCode("write_code") + conf = Config() + cost_manager = CostManager(conf.runtime_options) + llm = LLM(options=conf.runtime_options, cost_manager=cost_manager) + write_code = WriteCode(options=conf.runtime_options, name="write_code", llm=llm) - code = await write_code.run(api_design) + code = await write_code.run(api_design, "filename") logger.info(code) # 我们不能精确地预测生成的代码，但我们可以检查某些关键字 @@ -29,6 +34,7 @@ async def test_write_code(): @pytest.mark.asyncio async def test_write_code_directly(): prompt = WRITE_CODE_PROMPT_SAMPLE + '\n' + TASKS_2[0] - llm = LLM() + options = Config().runtime_options + llm = LLM(options=options, cost_manager=CostManager(options=options)) rsp = await llm.aask(prompt) logger.info(rsp) diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index 62a3a2361..457e665fa 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -1,8 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Desc : unittest of `metagpt/memory/longterm_memory.py` - -from metagpt.config import CONFIG +""" +@Desc : unittest of `metagpt/memory/longterm_memory.py` +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" +from metagpt.config import Config from metagpt.schema import Message from metagpt.actions import BossRequirement from metagpt.roles.role import RoleContext @@ -10,12 +12,13 @@ from metagpt.memory import LongTermMemory def test_ltm_search(): - assert hasattr(CONFIG, "long_term_memory") is True - openai_api_key = CONFIG.openai_api_key + conf = Config() + assert hasattr(conf, "long_term_memory") is True + openai_api_key = conf.openai_api_key assert len(openai_api_key) > 20 role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(watch=[BossRequirement]) + rc = RoleContext(options=conf.runtime_options, watch=[BossRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) @@ -23,19 +26,19 @@ def test_ltm_search(): message = Message(role='BOSS', content=idea, cause_by=BossRequirement) news = ltm.remember([message]) assert len(news) == 1 - ltm.add(message) + ltm.add(message, **conf.runtime_options) sim_idea = 'Write a game of cli snake' sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) news = ltm.remember([sim_message]) assert len(news) == 0 - ltm.add(sim_message) + ltm.add(sim_message, **conf.runtime_options) new_idea = 'Write a 2048 web game' new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) news = ltm.remember([new_message]) assert len(news) == 1 - ltm.add(new_message) + ltm.add(new_message, **conf.runtime_options) # restore from local index ltm_new = LongTermMemory() diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index a0f1f6257..d10c93ec0 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -4,14 +4,17 @@ @Time : 2023/5/12 00:47 @Author : alexanderwu @File : test_environment.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. + """ import pytest from metagpt.actions import BossRequirement +from metagpt.config import Config from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.manager import Manager +from metagpt.provider.openai_api import CostManager from metagpt.roles import Architect, ProductManager, Role from metagpt.schema import Message @@ -22,33 +25,45 @@ def env(): def test_add_role(env: Environment): - role = ProductManager("Alice", "product manager", "create a new product", "limited resources") + conf = Config() + cost_manager = CostManager(options=conf.runtime_options) + role = ProductManager(options=conf.runtime_options, + cost_manager=cost_manager, + name="Alice", + profile="product manager", + goal="create a new product", + constraints="limited resources") env.add_role(role) assert env.get_role(role.profile) == role def test_get_roles(env: Environment): - role1 = Role("Alice", "product manager", "create a new product", "limited resources") - role2 = Role("Bob", "engineer", "develop the new product", "short deadline") + conf = Config() + cost_manager = CostManager(options=conf.runtime_options) + role1 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", profile="product manager", + goal="create a new product", constraints="limited resources") + role2 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Bob", profile="engineer", + goal="develop the new product", constraints="short deadline") env.add_role(role1) env.add_role(role2) roles = env.get_roles() assert roles == {role1.profile: role1, role2.profile: role2} -def test_set_manager(env: Environment): - manager = Manager() - env.set_manager(manager) - assert env.manager == manager - - @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): - product_manager = ProductManager("Alice", "Product Manager", "做AI Native产品", "资源有限") - architect = Architect("Bob", "Architect", "设计一个可用、高效、较低成本的系统，包括数据结构与接口", "资源有限，需要节省成本") + conf = Config() + cost_manager = CostManager(options=conf.runtime_options) + product_manager = ProductManager(options=conf.runtime_options, + cost_manager=cost_manager, + name="Alice", profile="Product Manager", + goal="做AI Native产品", constraints="资源有限") + architect = Architect(options=conf.runtime_options, + cost_manager=cost_manager, + name="Bob", profile="Architect", goal="设计一个可用、高效、较低成本的系统，包括数据结构与接口", + constraints="资源有限，需要节省成本") env.add_roles([product_manager, architect]) - env.set_manager(Manager()) env.publish_message(Message(role="BOSS", content="需要一个基于LLM做总结的搜索引擎", cause_by=BossRequirement)) await env.run(k=2) diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 11503af1d..77de6df0c 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -4,16 +4,19 @@ @Time : 2023/5/11 14:45 @Author : alexanderwu @File : test_llm.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pytest -from metagpt.llm import LLM +from metagpt.config import Config +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager @pytest.fixture() def llm(): - return LLM() + options = Config().runtime_options + return LLM(options=options, cost_manager=CostManager(options)) @pytest.mark.asyncio diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index a7fe063a6..35ccdf78b 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -4,11 +4,13 @@ @Time : 2023/5/2 17:46 @Author : alexanderwu @File : test_search_engine.py +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import pytest +from metagpt.config import Config from metagpt.logs import logger from metagpt.tools import SearchEngineType from metagpt.tools.search_engine import SearchEngine @@ -37,9 +39,10 @@ class MockSearchEnine: ], ) -async def test_search_engine(search_engine_typpe, run_func, max_results, as_string, ): - search_engine = SearchEngine(search_engine_typpe, run_func) - rsp = await search_engine.run("metagpt", max_results=max_results, as_string=as_string) +async def test_search_engine(search_engine_typpe, run_func, max_results, as_string): + conf = Config() + search_engine = SearchEngine(options=conf.runtime_options, engine=search_engine_typpe, run_func=run_func) + rsp = await search_engine.run(query="metagpt", max_results=max_results, as_string=as_string) logger.info(rsp) if as_string: assert isinstance(rsp, str) diff --git a/tests/metagpt/tools/test_web_browser_engine.py b/tests/metagpt/tools/test_web_browser_engine.py index b08d0ca10..283633bd6 100644 --- a/tests/metagpt/tools/test_web_browser_engine.py +++ b/tests/metagpt/tools/test_web_browser_engine.py @@ -1,5 +1,10 @@ +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + import pytest +from metagpt.config import Config from metagpt.tools import WebBrowserEngineType, web_browser_engine @@ -13,7 +18,8 @@ from metagpt.tools import WebBrowserEngineType, web_browser_engine ids=["playwright", "selenium"], ) async def test_scrape_web_page(browser_type, url, urls): - browser = web_browser_engine.WebBrowserEngine(browser_type) + conf = Config() + browser = web_browser_engine.WebBrowserEngine(options=conf.runtime_options, engine=browser_type) result = await browser.run(url) assert isinstance(result, str) assert "深度赋智" in result diff --git a/tests/metagpt/tools/test_web_browser_engine_playwright.py b/tests/metagpt/tools/test_web_browser_engine_playwright.py index 69e1339e7..add2b2f63 100644 --- a/tests/metagpt/tools/test_web_browser_engine_playwright.py +++ b/tests/metagpt/tools/test_web_browser_engine_playwright.py @@ -1,6 +1,10 @@ +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + import pytest -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.tools import web_browser_engine_playwright @@ -15,22 +19,24 @@ from metagpt.tools import web_browser_engine_playwright ids=["chromium-normal", "firefox-normal", "webkit-normal"], ) async def test_scrape_web_page(browser_type, use_proxy, kwagrs, url, urls, proxy, capfd): + conf = Config() + global_proxy = conf.global_proxy try: - global_proxy = CONFIG.global_proxy if use_proxy: - CONFIG.global_proxy = proxy - browser = web_browser_engine_playwright.PlaywrightWrapper(browser_type, **kwagrs) + conf.global_proxy = proxy + browser = web_browser_engine_playwright.PlaywrightWrapper(options=conf.runtime_options, + browser_type=browser_type, **kwagrs) result = await browser.run(url) result = result.inner_text assert isinstance(result, str) - assert "Deepwisdom" in result + assert "DeepWisdom" in result if urls: results = await browser.run(url, *urls) assert isinstance(results, list) assert len(results) == len(urls) + 1 - assert all(("Deepwisdom" in i) for i in results) + assert all(("DeepWisdom" in i) for i in results) if use_proxy: assert "Proxy:" in capfd.readouterr().out finally: - CONFIG.global_proxy = global_proxy + conf.global_proxy = global_proxy diff --git a/tests/metagpt/tools/test_web_browser_engine_selenium.py b/tests/metagpt/tools/test_web_browser_engine_selenium.py index ce322f7bd..278c35c91 100644 --- a/tests/metagpt/tools/test_web_browser_engine_selenium.py +++ b/tests/metagpt/tools/test_web_browser_engine_selenium.py @@ -1,6 +1,10 @@ +""" +@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +""" + import pytest -from metagpt.config import CONFIG +from metagpt.config import Config from metagpt.tools import web_browser_engine_selenium @@ -15,11 +19,12 @@ from metagpt.tools import web_browser_engine_selenium ids=["chrome-normal", "firefox-normal", "edge-normal"], ) async def test_scrape_web_page(browser_type, use_proxy, url, urls, proxy, capfd): + conf = Config() + global_proxy = conf.global_proxy try: - global_proxy = CONFIG.global_proxy if use_proxy: - CONFIG.global_proxy = proxy - browser = web_browser_engine_selenium.SeleniumWrapper(browser_type) + conf.global_proxy = proxy + browser = web_browser_engine_selenium.SeleniumWrapper(options=conf.runtime_options, browser_type=browser_type) result = await browser.run(url) result = result.inner_text assert isinstance(result, str) @@ -33,4 +38,4 @@ async def test_scrape_web_page(browser_type, use_proxy, url, urls, proxy, capfd) if use_proxy: assert "Proxy:" in capfd.readouterr().out finally: - CONFIG.global_proxy = global_proxy + conf.global_proxy = global_proxy diff --git a/tests/metagpt/utils/test_config.py b/tests/metagpt/utils/test_config.py index 475bac22b..510892c2f 100644 --- a/tests/metagpt/utils/test_config.py +++ b/tests/metagpt/utils/test_config.py @@ -4,7 +4,7 @@ @Time : 2023/5/1 11:19 @Author : alexanderwu @File : test_config.py -@Modified By: mashenquan, 2013/8/20, add `test_options` +@Modified By: mashenquan, 2013/8/20, Add `test_options`; remove global configuration `CONFIG`, enable configuration support for business isolation. """ from pathlib import Path @@ -13,12 +13,6 @@ import pytest from metagpt.config import Config -def test_config_class_is_singleton(): - config_1 = Config() - config_2 = Config() - assert config_1 == config_2 - - def test_config_class_get_key_exception(): with pytest.raises(Exception) as exc_info: config = Config() @@ -27,16 +21,15 @@ def test_config_class_get_key_exception(): def test_config_yaml_file_not_exists(): - config = Config('wtf.yaml') with pytest.raises(Exception) as exc_info: - config.get('OPENAI_BASE_URL') - assert str(exc_info.value) == "Key 'OPENAI_BASE_URL' not found in environment variables or in the YAML file" + Config(Path('wtf.yaml')) + assert str(exc_info.value) == "Set OPENAI_API_KEY or Anthropic_API_KEY first" def test_options(): filename = Path(__file__).resolve().parent.parent.parent.parent / "config/config.yaml" config = Config(filename) - opts = config.options + opts = config.runtime_options assert opts From 88da7aa76145b9dd01e9d26f60afeebd3bc1ec5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 19:23:05 +0800 Subject: [PATCH 059/378] feat: +skill meta data decorator --- metagpt/learn/skill_metadata.py | 25 +++++++++++++++++++++++++ metagpt/learn/text_to_embedding.py | 4 ++++ metagpt/learn/text_to_image.py | 4 ++++ metagpt/learn/text_to_speech.py | 4 ++++ 4 files changed, 37 insertions(+) create mode 100644 metagpt/learn/skill_metadata.py diff --git a/metagpt/learn/skill_metadata.py b/metagpt/learn/skill_metadata.py new file mode 100644 index 000000000..6a13d6274 --- /dev/null +++ b/metagpt/learn/skill_metadata.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/20 +@Author : mashenquan +@File : skill_metadata.py +@Desc : Defines metadata for the `skill`. + Depending on the context and specific circumstances, skills may have different effects. + For example: + Proprietor: "Skill of the proprietor entity."（所有者的技能） + Holder: "Skill of the holder entity."（持有者的技能） + Possessor: "Skill of the possessor entity."（拥有者的技能） + Controller: "Skill of the controller entity."（控制者的技能） + Owner: "Skill of the owner entity."（所有者的技能） +""" + + +def skill_metadata(name, description, requisite): + def decorator(func): + func.skill_name = name + func.skill_description = description + func.skill_requisite = requisite + return func + + return decorator diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 281815ca6..38fd7c0cb 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -8,10 +8,14 @@ """ import os +from metagpt.learn.skill_metadata import skill_metadata from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding from metagpt.utils.common import initialize_environment +@skill_metadata(name="Text to Embedding", + description="Convert the text into embeddings.", + requisite="`OPENAI_API_KEY`") def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): """Text to embedding diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 0932dfe07..d123e116a 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -8,11 +8,15 @@ """ import os +from metagpt.learn.skill_metadata import skill_metadata from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image from metagpt.utils.common import initialize_environment +@skill_metadata(name="Text to image", + description="Create a drawing based on the text.", + requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url=""): """Text to image diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 1b81097b8..5631ef45e 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -8,10 +8,14 @@ """ import os +from metagpt.learn.skill_metadata import skill_metadata from metagpt.tools.azure_tts import oas3_azsure_tts from metagpt.utils.common import initialize_environment +@skill_metadata(name="Text to speech", + description="Text-to-speech", + requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region=""): """Text to speech From c41f16e7bc58a3df13f04cdf000f4d41c580df76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 19:24:10 +0800 Subject: [PATCH 060/378] feat: +skill meta data decorator --- metagpt/learn/skill_metadata.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/learn/skill_metadata.py b/metagpt/learn/skill_metadata.py index 6a13d6274..dea5fb04d 100644 --- a/metagpt/learn/skill_metadata.py +++ b/metagpt/learn/skill_metadata.py @@ -7,11 +7,11 @@ @Desc : Defines metadata for the `skill`. Depending on the context and specific circumstances, skills may have different effects. For example: - Proprietor: "Skill of the proprietor entity."（所有者的技能） - Holder: "Skill of the holder entity."（持有者的技能） - Possessor: "Skill of the possessor entity."（拥有者的技能） - Controller: "Skill of the controller entity."（控制者的技能） - Owner: "Skill of the owner entity."（所有者的技能） + Proprietor: "Skill of the proprietor entity." + Holder: "Skill of the holder entity." + Possessor: "Skill of the possessor entity." + Controller: "Skill of the controller entity." + Owner: "Skill of the owner entity." """ From ae94b6dff8e10cb65450bd05a8acf14ee24a169d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 20 Aug 2023 20:22:59 +0800 Subject: [PATCH 061/378] feat: merge role_option --- metagpt/actions/write_teaching_plan.py | 4 ++-- metagpt/roles/fork_meta_role.py | 11 +++++++---- metagpt/roles/role.py | 1 + metagpt/roles/teacher.py | 6 +++--- .../metagpt/actions/test_write_teaching_plan.py | 8 +++++--- tests/metagpt/roles/test_fork_meta_role.py | 6 +++++- tests/metagpt/roles/test_teacher.py | 17 +++++++++++------ 7 files changed, 34 insertions(+), 19 deletions(-) diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 3718c9801..53371b5a1 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -20,7 +20,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): + def __init__(self, options, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ :param name: action name @@ -29,7 +29,7 @@ class WriteTeachingPlanPart(Action): :param topic: topic part of teaching plan :param language: A human language, such as Chinese, English, French, etc. """ - super().__init__(name, context, llm) + super().__init__(options, name, context, llm) self.topic = topic self.language = language self.rsp = None diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index 555bc8cf3..c21d08e37 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -26,14 +26,16 @@ from metagpt.schema import Message class ForkMetaRole(Role): """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" - def __init__(self, options, **kwargs): + def __init__(self, runtime_options, cost_manager, role_options, **kwargs): """Initialize a `fork` style meta role - :param options: pattern yaml file data + :param runtime_options: System configuration + :param cost_manager: Cost manager + :param role_options: pattern yaml file data :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` """ - opts = UMLMetaRoleOptions(**options) + opts = UMLMetaRoleOptions(**role_options) global_variables = { "name": Role.format_value(opts.name, kwargs), "profile": Role.format_value(opts.profile, kwargs), @@ -47,6 +49,8 @@ class ForkMetaRole(Role): global_variables[k] = v super(ForkMetaRole, self).__init__( + options=runtime_options, + cost_manager=cost_manager, name=global_variables["name"], profile=global_variables["profile"], goal=global_variables["goal"], @@ -54,7 +58,6 @@ class ForkMetaRole(Role): desc=global_variables["desc"], **kwargs ) - self.options = options actions = [] for m in opts.actions: for k, v in m.items(): diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5397893eb..00f8ed45f 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -173,6 +173,7 @@ class Role: """Return number of action""" return len(self._actions) + @property def options(self): return self._options diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 24ede7402..f29f384db 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -20,13 +20,13 @@ import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, name='Lily', profile='{teaching_language} Teacher', + def __init__(self, options, name='Lily', profile='{teaching_language} Teacher', goal='writing a {language} teaching plan part by part', constraints='writing in {language}', desc="", *args, **kwargs): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + super().__init__(options=options, name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: - act = WriteTeachingPlanPart(topic=topic, llm=self._llm) + act = WriteTeachingPlanPart(options=options, topic=topic, llm=self._llm) actions.append(act) self._init_actions(actions) self._watch({TeachingPlanRequirement}) diff --git a/tests/metagpt/actions/test_write_teaching_plan.py b/tests/metagpt/actions/test_write_teaching_plan.py index 299a89639..6754fe88c 100644 --- a/tests/metagpt/actions/test_write_teaching_plan.py +++ b/tests/metagpt/actions/test_write_teaching_plan.py @@ -12,12 +12,13 @@ from pydantic import BaseModel from langchain.llms.base import LLM from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart +from metagpt.config import Config from metagpt.schema import Message class MockWriteTeachingPlanPart(WriteTeachingPlanPart): - def __init__(self, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): - super().__init__(name, context, llm, topic, language) + def __init__(self, options, name: str = '', context=None, llm: LLM = None, topic="", language="Chinese"): + super().__init__(options, name, context, llm, topic, language) async def _aask(self, prompt: str, system_msgs: Optional[list[str]] = None) -> str: return f"{WriteTeachingPlanPart.DATA_BEGIN_TAG}\nprompt\n{WriteTeachingPlanPart.DATA_END_TAG}" @@ -47,7 +48,8 @@ async def mock_write_teaching_plan_part(): for i in inputs: seed = Inputs(**i) - act = MockWriteTeachingPlanPart(name=seed.name, topic=seed.topic, language=seed.language) + options = Config().runtime_options + act = MockWriteTeachingPlanPart(options=options, name=seed.name, topic=seed.topic, language=seed.language) await act.run([Message(content="")]) assert act.topic == seed.topic assert str(act) == seed.topic diff --git a/tests/metagpt/roles/test_fork_meta_role.py b/tests/metagpt/roles/test_fork_meta_role.py index b2659330d..355197234 100644 --- a/tests/metagpt/roles/test_fork_meta_role.py +++ b/tests/metagpt/roles/test_fork_meta_role.py @@ -9,6 +9,8 @@ from typing import Dict from pydantic import BaseModel +from metagpt.config import Config +from metagpt.provider.openai_api import CostManager from metagpt.roles.fork_meta_role import ForkMetaRole @@ -79,7 +81,9 @@ def test_creat_role(): "teaching_language": "AA", "language": "BB" } - role = ForkMetaRole(seed.role, **kwargs) + runtime_options = Config().runtime_options + cost_manager = CostManager(options=runtime_options) + role = ForkMetaRole(runtime_options=runtime_options, cost_manager=cost_manager, role_options=seed.role, **kwargs) assert role.action_count == 2 assert "{" not in role.profile assert "{" not in role.goal diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 5faa43455..11c268edb 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -9,6 +9,8 @@ from typing import Dict, Optional from pydantic import BaseModel +from metagpt.config import Config +from metagpt.provider.openai_api import CostManager from metagpt.roles.teacher import Teacher @@ -42,22 +44,25 @@ def test_init(): }, { "name": "Lily{language}", - "expect_name": "LilyChinese", + "expect_name": "Lily{language}", "profile": "X {teaching_language}", - "expect_profile": "X English", + "expect_profile": "X {teaching_language}", "goal": "Do {something_big}, {language}", - "expect_goal": "Do {something_big}, Chinese", + "expect_goal": "Do {something_big}, {language}", "constraints": "Do in {key1}, {language}", - "expect_constraints": "Do in {key1}, Chinese", + "expect_constraints": "Do in {key1}, {language}", "kwargs": {}, "desc": "aaa{language}", - "expect_desc": "aaaChinese" + "expect_desc": "aaa{language}" }, ] for i in inputs: seed = Inputs(**i) - teacher = Teacher(name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, + options = Config().runtime_options + cost_manager = CostManager(options=options) + teacher = Teacher(options=options, cost_manager=cost_manager, name=seed.name, profile=seed.profile, + goal=seed.goal, constraints=seed.constraints, desc=seed.desc, **seed.kwargs) assert teacher.name == seed.expect_name assert teacher.desc == seed.expect_desc From 86e0e706191dc8822d1ed183108fc6546175d16b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 21 Aug 2023 10:44:40 +0800 Subject: [PATCH 062/378] fixbug: teacher role --- .../fork_meta_role_write_teaching_plan.py | 5 +++- examples/write_teaching_plan.py | 2 +- metagpt/actions/meta_action.py | 25 +++++++++++-------- metagpt/roles/fork_meta_role.py | 8 +++--- metagpt/roles/uml_meta_role_factory.py | 2 +- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/examples/fork_meta_role_write_teaching_plan.py b/examples/fork_meta_role_write_teaching_plan.py index d2898605e..e529a9b46 100644 --- a/examples/fork_meta_role_write_teaching_plan.py +++ b/examples/fork_meta_role_write_teaching_plan.py @@ -90,8 +90,11 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * configs = yaml.safe_load(reader) startup_config = ProjectConfig(**configs) - roles = UMLMetaRoleFactory.create_roles(startup_config.roles, **kwargs) company = SoftwareCompany() + roles = UMLMetaRoleFactory.create_roles(role_configs=startup_config.roles, + options=company.options, + cost_manager=company.cost_manager, + **kwargs) company.hire(roles) company.invest(startup_config.startup.investment) company.start_project(lesson, role=startup_config.startup.role, diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 9874d10a5..6ab5edce4 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -77,7 +77,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * lesson = demo_lesson company = SoftwareCompany() - company.hire([Teacher(*args, **kwargs)]) + company.hire([Teacher(options=company.options, cost_manager=company.cost_manager, *args, **kwargs)]) company.invest(investment) company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) await company.run(n_round=1) diff --git a/metagpt/actions/meta_action.py b/metagpt/actions/meta_action.py index 3f01b8c0f..4c52e7cfd 100644 --- a/metagpt/actions/meta_action.py +++ b/metagpt/actions/meta_action.py @@ -21,19 +21,22 @@ from metagpt.schema import Message class MetaAction(Action): - def __init__(self, options: MetaActionOptions, llm=None, **kwargs): - super(MetaAction, self).__init__(options.name, kwargs.get("context"), llm=llm) - self.prompt = options.format_prompt(**kwargs) - self.options = options + def __init__(self, options, action_options: MetaActionOptions, llm=None, **kwargs): + super(MetaAction, self).__init__(options=options, + name=action_options.name, + context=kwargs.get("context"), + llm=llm) + self.prompt = action_options.format_prompt(**kwargs) + self.action_options = action_options self.kwargs = kwargs def __str__(self): """Return `topic` value when str()""" - return self.options.topic + return self.action_options.topic def __repr__(self): """Show `topic` value when debug""" - return self.options.topic + return self.action_options.topic async def run(self, messages, *args, **kwargs): if len(messages) < 1 or not isinstance(messages[0], Message): @@ -46,11 +49,11 @@ class MetaAction(Action): return self.rsp def _set_result(self, rsp): - if self.options.rsp_begin_tag and self.options.rsp_begin_tag in rsp: - ix = rsp.index(self.options.rsp_begin_tag) - rsp = rsp[ix + len(self.options.rsp_begin_tag):] - if self.options.rsp_end_tag and self.options.rsp_end_tag in rsp: - ix = rsp.index(self.options.rsp_end_tag) + if self.action_options.rsp_begin_tag and self.action_options.rsp_begin_tag in rsp: + ix = rsp.index(self.action_options.rsp_begin_tag) + rsp = rsp[ix + len(self.action_options.rsp_begin_tag):] + if self.action_options.rsp_end_tag and self.action_options.rsp_end_tag in rsp: + ix = rsp.index(self.action_options.rsp_end_tag) rsp = rsp[0:ix] self.rsp = rsp.strip() diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index c21d08e37..5311bc4f0 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -26,10 +26,10 @@ from metagpt.schema import Message class ForkMetaRole(Role): """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" - def __init__(self, runtime_options, cost_manager, role_options, **kwargs): + def __init__(self, options, cost_manager, role_options, **kwargs): """Initialize a `fork` style meta role - :param runtime_options: System configuration + :param options: System configuration :param cost_manager: Cost manager :param role_options: pattern yaml file data :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` @@ -49,7 +49,7 @@ class ForkMetaRole(Role): global_variables[k] = v super(ForkMetaRole, self).__init__( - options=runtime_options, + options=options, cost_manager=cost_manager, name=global_variables["name"], profile=global_variables["profile"], @@ -70,7 +70,7 @@ class ForkMetaRole(Role): o = MetaActionOptions(**m) o.set_default_template(opts.templates[o.template_ix]) - act = MetaAction(options=o, llm=self._llm, **m) + act = MetaAction(options=options, action_options=o, llm=self._llm, **m) actions.append(act) self._init_actions(actions) requirement_types = set() diff --git a/metagpt/roles/uml_meta_role_factory.py b/metagpt/roles/uml_meta_role_factory.py index 78f9689a2..42071b0a6 100644 --- a/metagpt/roles/uml_meta_role_factory.py +++ b/metagpt/roles/uml_meta_role_factory.py @@ -33,7 +33,7 @@ class UMLMetaRoleFactory: raise NotImplementedError( f"{opt.role_type} is not implemented" ) - r = constructor(m, **kwargs) + r = constructor(role_options=m, **kwargs) roles.append(r) return roles From 58b1acf7b935ad0104fdc65a8133b7131de45dcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 21 Aug 2023 21:30:37 +0800 Subject: [PATCH 063/378] feat: +Message + tags --- metagpt/schema.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 27f5dd10c..4e6cba4ca 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -8,7 +8,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Type, TypedDict +from typing import Type, TypedDict, Set from pydantic import BaseModel @@ -29,6 +29,7 @@ class Message: cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") + tags: Set = field(default_factory=Set) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From cf225320eb69ca2dfeca71730ec48022203f2faf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 10:22:19 +0800 Subject: [PATCH 064/378] feat: +Message to __init__ --- metagpt/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/__init__.py b/metagpt/__init__.py index b9c530d24..7e0247553 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -3,3 +3,9 @@ # @Time : 2023/4/24 22:26 # @Author : alexanderwu # @File : __init__.py + +from metagpt.schema import Message + +__all__ = [ + "Message", +] From 5121472bd85e9cac565cc08bf5c763a00de522fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 10:23:42 +0800 Subject: [PATCH 065/378] feat: +Message to __init__ --- metagpt/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/__init__.py b/metagpt/__init__.py index 7e0247553..16359ca19 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -3,6 +3,7 @@ # @Time : 2023/4/24 22:26 # @Author : alexanderwu # @File : __init__.py +# @Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. from metagpt.schema import Message From 148279401ee3c10b991df95f0a078e28f51a73ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 10:59:26 +0800 Subject: [PATCH 066/378] feat: Add tags to enable custom message classification --- metagpt/__init__.py | 10 ++++++---- metagpt/schema.py | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/__init__.py b/metagpt/__init__.py index 16359ca19..2980109dd 100644 --- a/metagpt/__init__.py +++ b/metagpt/__init__.py @@ -1,9 +1,11 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# @Time : 2023/4/24 22:26 -# @Author : alexanderwu -# @File : __init__.py -# @Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. +""" +@Time : 2023/4/24 22:26 +@Author : alexanderwu +@File : __init__.py +@Desc : mashenquan, 2023/8/22. Add `Message` for importing by external projects. +""" from metagpt.schema import Message diff --git a/metagpt/schema.py b/metagpt/schema.py index 4e6cba4ca..749e0fd56 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -4,6 +4,7 @@ @Time : 2023/5/8 22:12 @Author : alexanderwu @File : schema.py +@Desc : mashenquan, 2023/8/22. Add tags to enable custom message classification. """ from __future__ import annotations @@ -29,7 +30,7 @@ class Message: cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") - tags: Set = field(default_factory=Set) + tags: Set = field(default_factory=set()) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From 2adcefc298918101d7a50e2a785154ef69b96b6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 11:04:29 +0800 Subject: [PATCH 067/378] feat: Add tags to enable custom message classification --- metagpt/schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 749e0fd56..2e4a6c62f 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Type, TypedDict, Set +from typing import Type, TypedDict, Set, Optional from pydantic import BaseModel @@ -30,7 +30,7 @@ class Message: cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") - tags: Set = field(default_factory=set()) + tags: Optional[Set] = field(default=None) def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From bc97b709bb17e7d25cc48f49632648ff5cb32624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 11:10:08 +0800 Subject: [PATCH 068/378] feat: Add tags to enable custom message classification --- metagpt/schema.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 2e4a6c62f..140f207c8 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -45,6 +45,16 @@ class Message: "content": self.content } + def add_tag(self, tag): + if self.tags is None: + self.tags = set() + self.tags.add(tag) + + def remove_tag(self, tag): + if self.tags is None: + return + self.tags.remove(tag) + @dataclass class UserMessage(Message): From a2e9797d4e7f7af85f43d6f8bf686181a93cc402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 11:13:08 +0800 Subject: [PATCH 069/378] feat: Add tags to enable custom message classification --- metagpt/schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 140f207c8..0119f5bbb 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -51,7 +51,7 @@ class Message: self.tags.add(tag) def remove_tag(self, tag): - if self.tags is None: + if self.tags is None or tag not in self.tags: return self.tags.remove(tag) From 8eaf22dd62e47b4cc7611cd1b2fa2338a0af3ca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 18:49:39 +0800 Subject: [PATCH 070/378] fixbug: role option, cost_manager argments --- metagpt/roles/customer_service.py | 4 +++- metagpt/roles/researcher.py | 4 +++- metagpt/roles/sales.py | 4 +++- metagpt/roles/seacher.py | 4 ++-- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 4aae7cb03..8550313d4 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -26,9 +26,11 @@ DESC = """ class CustomerService(Sales): def __init__( self, + options, + cost_manager, name="Xiaomei", profile="Human customer service", desc=DESC, store=None ): - super().__init__(name, profile, desc=desc, store=store) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc, store=store) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index acb46c718..6d8d072d9 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -22,6 +22,8 @@ class Report(BaseModel): class Researcher(Role): def __init__( self, + options, + cost_manager, name: str = "David", profile: str = "Researcher", goal: str = "Gather information and conduct research", @@ -29,7 +31,7 @@ class Researcher(Role): language: str = "en-us", **kwargs, ): - super().__init__(name, profile, goal, constraints, **kwargs) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) self.language = language if language not in ("en-us", "zh-cn"): diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index 51b13f487..35146fdc3 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -13,6 +13,8 @@ from metagpt.tools import SearchEngineType class Sales(Role): def __init__( self, + options, + cost_manager, name="Xiaomei", profile="Retail sales guide", desc="I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " @@ -23,7 +25,7 @@ class Sales(Role): "professional guide", store=None ): - super().__init__(name, profile, desc=desc) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc) self._set_store(store) def _set_store(self, store): diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index c116ce98b..7b07ce713 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -13,9 +13,9 @@ from metagpt.tools import SearchEngineType class Searcher(Role): - def __init__(self, name='Alice', profile='Smart Assistant', goal='Provide search services for users', + def __init__(self, options, cost_manager, name='Alice', profile='Smart Assistant', goal='Provide search services for users', constraints='Answer is rich and complete', engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs): - super().__init__(name, profile, goal, constraints, **kwargs) + super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) self._init_actions([SearchAndSummarize(engine=engine)]) def set_search_func(self, search_func): From 9600787d63b7575edac30e505cff503b5c95e424 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 18:56:23 +0800 Subject: [PATCH 071/378] fixbug: role option, cost_manager argments --- metagpt/schema.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 0119f5bbb..f45d1e36d 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,6 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from enum import StrEnum from typing import Type, TypedDict, Set, Optional from pydantic import BaseModel @@ -16,6 +17,10 @@ from pydantic import BaseModel from metagpt.logs import logger +class MessageTag(StrEnum): + Prerequisite = "prerequisite" + + class RawMessage(TypedDict): content: str role: str @@ -61,6 +66,7 @@ class UserMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ + def __init__(self, content: str): super().__init__(content, 'user') @@ -70,6 +76,7 @@ class SystemMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ + def __init__(self, content: str): super().__init__(content, 'system') @@ -79,6 +86,7 @@ class AIMessage(Message): """便于支持OpenAI的消息 Facilitate support for OpenAI messages """ + def __init__(self, content: str): super().__init__(content, 'assistant') From 19767496b16bd05119254c60215093a90c27a6d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 19:47:35 +0800 Subject: [PATCH 072/378] =?UTF-8?q?feat:=20CostManager=E6=94=B9pydantic?= =?UTF-8?q?=E7=BB=93=E6=9E=84=EF=BC=8C=E4=BB=A5=E5=A4=87RPC=E4=BC=A0?= =?UTF-8?q?=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/base_gpt_api.py | 7 +++++- metagpt/provider/openai_api.py | 32 ++++++++---------------- metagpt/schema.py | 4 +-- metagpt/software_company.py | 2 +- tests/metagpt/actions/test_write_code.py | 4 +-- tests/metagpt/test_environment.py | 6 ++--- tests/metagpt/test_llm.py | 2 +- 7 files changed, 26 insertions(+), 31 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f39e708eb..f1590a77c 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -4,6 +4,7 @@ @Time : 2023/5/5 23:04 @Author : alexanderwu @File : base_gpt_api.py +@Desc : mashenquan, 2023/8/22. + try catch """ from abc import abstractmethod from typing import Optional @@ -41,7 +42,11 @@ class BaseGPTAPI(BaseChatbot): message = self._system_msgs(system_msgs) + [self._user_msg(msg)] else: message = [self._default_system_msg(), self._user_msg(msg)] - rsp = await self.acompletion_text(message, stream=True) + try: + rsp = await self.acompletion_text(message, stream=True) + except Exception as e: + logger.exception(f"{e}") + raise e logger.debug(message) # logger.debug(rsp) return rsp diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2e951b36f..abfb796f3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -8,10 +8,11 @@ """ import asyncio import time -from typing import NamedTuple +from typing import NamedTuple, Dict import openai from openai.error import APIConnectionError +from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.logs import logger @@ -35,7 +36,7 @@ class RateLimiter: self.rpm = rpm def split_batches(self, batch): - return [batch[i : i + self.rpm] for i in range(0, len(batch), self.rpm)] + return [batch[i: i + self.rpm] for i in range(0, len(batch), self.rpm)] async def wait_if_needed(self, num_requests): current_time = time.time() @@ -56,14 +57,14 @@ class Costs(NamedTuple): total_budget: float -class CostManager: +class CostManager(BaseModel): """计算使用接口的开销""" - def __init__(self, options): - self.total_prompt_tokens = 0 - self.total_completion_tokens = 0 - self.options = options - self.total_budget = 0 + total_prompt_tokens: int = 0 + total_completion_tokens: int = 0 + total_budget: int = 0 + max_budget: int + total_cost: int = 0 def update_cost(self, prompt_tokens, completion_tokens, model): """ @@ -76,7 +77,8 @@ class CostManager: """ self.total_prompt_tokens += prompt_tokens self.total_completion_tokens += completion_tokens - cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model]["completion"]) / 1000 + cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ + "completion"]) / 1000 self.total_cost += cost logger.info( f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " @@ -114,18 +116,6 @@ class CostManager: """获得所有开销""" return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) - @property - def total_cost(self): - return self.options.get("total_cost", 0) - - @total_cost.setter - def total_cost(self, v): - self.options["total_cost"] = v - - @property - def max_budget(self): - return self.options.get("max_budget", 0) - def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") diff --git a/metagpt/schema.py b/metagpt/schema.py index f45d1e36d..56e9ad95c 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -9,7 +9,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from enum import StrEnum +from enum import Enum from typing import Type, TypedDict, Set, Optional from pydantic import BaseModel @@ -17,7 +17,7 @@ from pydantic import BaseModel from metagpt.logs import logger -class MessageTag(StrEnum): +class MessageTag(Enum): Prerequisite = "prerequisite" diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 3f6f484b4..87b24a1cb 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -30,7 +30,7 @@ class SoftwareCompany(BaseModel): investment: float = Field(default=10.0) idea: str = Field(default="") options: Dict = Field(default=Config().runtime_options) - cost_manager: CostManager = Field(default=CostManager(Config().runtime_options)) + cost_manager: CostManager = Field(default=CostManager(**Config().runtime_options)) class Config: arbitrary_types_allowed = True diff --git a/tests/metagpt/actions/test_write_code.py b/tests/metagpt/actions/test_write_code.py index 04216ad7c..9861fd4cd 100644 --- a/tests/metagpt/actions/test_write_code.py +++ b/tests/metagpt/actions/test_write_code.py @@ -19,7 +19,7 @@ from tests.metagpt.actions.mock import TASKS_2, WRITE_CODE_PROMPT_SAMPLE async def test_write_code(): api_design = "设计一个名为'add'的函数，该函数接受两个整数作为输入，并返回它们的和。" conf = Config() - cost_manager = CostManager(conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) llm = LLM(options=conf.runtime_options, cost_manager=cost_manager) write_code = WriteCode(options=conf.runtime_options, name="write_code", llm=llm) @@ -35,6 +35,6 @@ async def test_write_code(): async def test_write_code_directly(): prompt = WRITE_CODE_PROMPT_SAMPLE + '\n' + TASKS_2[0] options = Config().runtime_options - llm = LLM(options=options, cost_manager=CostManager(options=options)) + llm = LLM(options=options, cost_manager=CostManager(**options)) rsp = await llm.aask(prompt) logger.info(rsp) diff --git a/tests/metagpt/test_environment.py b/tests/metagpt/test_environment.py index d10c93ec0..57650d145 100644 --- a/tests/metagpt/test_environment.py +++ b/tests/metagpt/test_environment.py @@ -26,7 +26,7 @@ def env(): def test_add_role(env: Environment): conf = Config() - cost_manager = CostManager(options=conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) role = ProductManager(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", @@ -39,7 +39,7 @@ def test_add_role(env: Environment): def test_get_roles(env: Environment): conf = Config() - cost_manager = CostManager(options=conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) role1 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", profile="product manager", goal="create a new product", constraints="limited resources") role2 = Role(options=conf.runtime_options, cost_manager=cost_manager, name="Bob", profile="engineer", @@ -53,7 +53,7 @@ def test_get_roles(env: Environment): @pytest.mark.asyncio async def test_publish_and_process_message(env: Environment): conf = Config() - cost_manager = CostManager(options=conf.runtime_options) + cost_manager = CostManager(**conf.runtime_options) product_manager = ProductManager(options=conf.runtime_options, cost_manager=cost_manager, name="Alice", profile="Product Manager", diff --git a/tests/metagpt/test_llm.py b/tests/metagpt/test_llm.py index 77de6df0c..f61793151 100644 --- a/tests/metagpt/test_llm.py +++ b/tests/metagpt/test_llm.py @@ -16,7 +16,7 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager @pytest.fixture() def llm(): options = Config().runtime_options - return LLM(options=options, cost_manager=CostManager(options)) + return LLM(options=options, cost_manager=CostManager(**options)) @pytest.mark.asyncio From a7157d9e7a0d7c3cbf3a248e32d18ebec2c90fd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 19:56:22 +0800 Subject: [PATCH 073/378] =?UTF-8?q?feat:=20CostManager=E6=94=B9pydantic?= =?UTF-8?q?=E7=BB=93=E6=9E=84=EF=BC=8C=E4=BB=A5=E5=A4=87RPC=E4=BC=A0?= =?UTF-8?q?=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/openai_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index abfb796f3..f0b692f46 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -62,9 +62,9 @@ class CostManager(BaseModel): total_prompt_tokens: int = 0 total_completion_tokens: int = 0 - total_budget: int = 0 - max_budget: int - total_cost: int = 0 + total_budget: float = 0 + max_budget: float + total_cost: float = 0 def update_cost(self, prompt_tokens, completion_tokens, model): """ From 6e37e156de17254fccba4ff4dddba6e9e604f899 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 22 Aug 2023 21:13:24 +0800 Subject: [PATCH 074/378] fixbug: init action error --- metagpt/roles/researcher.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 6d8d072d9..30545c5c0 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -32,7 +32,10 @@ class Researcher(Role): **kwargs, ): super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) - self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) + self._init_actions([ + CollectLinks(options=options, name=name), + WebBrowseAndSummarize(options=options, name=name), + ConductResearch(options=options, name=name)]) self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") From 937bd12a63733d818338f7d3ad8c2d0907fe5c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 13:02:23 +0800 Subject: [PATCH 075/378] feat: memory + tags --- metagpt/memory/memory.py | 8 ++++++++ metagpt/roles/role.py | 9 +++++++-- metagpt/schema.py | 7 +++++++ tests/metagpt/roles/test_teacher.py | 2 +- 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 625d98675..1a8003fba 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -91,3 +91,11 @@ class Memory: key = class_names[type(action).__name__] rsp += self.index[key] return rsp + + def get_by_tags(self, tags: list) -> list[Message]: + """Return messages with specified tags""" + result = [] + for m in self.storage: + if m.is_contain_tags(tags): + result.append(m) + return result diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 00f8ed45f..217272b54 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -17,7 +17,7 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as LLM from metagpt.actions import Action, ActionOutput from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory -from metagpt.schema import Message +from metagpt.schema import Message, MessageTag PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -90,6 +90,11 @@ class RoleContext(BaseModel): def history(self) -> list[Message]: return self.memory.get() + @property + def prerequisite(self): + """Retrieve information with `prerequisite` tag""" + return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + class Role: """Role/Proxy""" @@ -209,7 +214,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - requirement = self._rc.important_memory + requirement = self._rc.important_memory or self._rc.prerequisite response = await self._rc.todo.run(requirement, **self._options) # logger.info(response) if isinstance(response, ActionOutput): diff --git a/metagpt/schema.py b/metagpt/schema.py index 56e9ad95c..4c577fd7b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -60,6 +60,13 @@ class Message: return self.tags.remove(tag) + def is_contain_tags(self, tags: list) -> bool: + """Determine whether the message contains tags.""" + if not tags or not self.tags: + return False + intersection = set(tags) & self.tags + return len(intersection) > 0 + @dataclass class UserMessage(Message): diff --git a/tests/metagpt/roles/test_teacher.py b/tests/metagpt/roles/test_teacher.py index 11c268edb..8f673d6e0 100644 --- a/tests/metagpt/roles/test_teacher.py +++ b/tests/metagpt/roles/test_teacher.py @@ -60,7 +60,7 @@ def test_init(): for i in inputs: seed = Inputs(**i) options = Config().runtime_options - cost_manager = CostManager(options=options) + cost_manager = CostManager(**options) teacher = Teacher(options=options, cost_manager=cost_manager, name=seed.name, profile=seed.profile, goal=seed.goal, constraints=seed.constraints, desc=seed.desc, **seed.kwargs) From 9395d9f7dc5ee0a8b1587ce74afd2798b0e098ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 14:56:53 +0800 Subject: [PATCH 076/378] feat: Add options to Config.__init__ to support externally specified options. --- metagpt/config.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 076bc5eb7..d8d772cd0 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -3,8 +3,9 @@ """ @Desc: Provide configuration, singleton. @Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config -@Desc: `os.environ` doesn't support personalization, while `Config` does. + `os.environ` doesn't support personalization, while `Config` does. Hence, the parameter reading priority is `Config` first, and if not found, then `os.environ`. +@Modified By: mashenquan, 2023/8/23. Add `options` to `Config.__init__` to support externally specified options. """ import os @@ -43,10 +44,14 @@ class Config: key_yaml_file = PROJECT_ROOT / "config/key.yaml" default_yaml_file = PROJECT_ROOT / "config/config.yaml" - def __init__(self, yaml_file=default_yaml_file): + def __init__(self, yaml_file=default_yaml_file, options=None): self._configs = {} self._init_with_config_files_and_env(self._configs, yaml_file) + if options: + self._configs.update(options) + self._parse() + def _parse(self): logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") From 7dd02ae4b11a5494a470125f785e4acbf7406b7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 15:53:33 +0800 Subject: [PATCH 077/378] feat: A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. --- metagpt/roles/fork_meta_role.py | 5 ++++- metagpt/roles/researcher.py | 9 +++++++-- metagpt/roles/role.py | 8 +++++--- metagpt/roles/teacher.py | 7 +++++-- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py index 5311bc4f0..57d467080 100644 --- a/metagpt/roles/fork_meta_role.py +++ b/metagpt/roles/fork_meta_role.py @@ -10,6 +10,8 @@ For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + """ import re @@ -82,12 +84,13 @@ class ForkMetaRole(Role): """Everything will be done part by part.""" if self._rc.todo is None: self._set_state(0) - return + return True if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) else: self._rc.todo = None + return False async def _react(self) -> Message: ret = Message(content="") diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index 30545c5c0..f3ff7f8e5 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -1,4 +1,8 @@ #!/usr/bin/env python +""" +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + +""" import asyncio @@ -40,15 +44,16 @@ class Researcher(Role): if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") - async def _think(self) -> None: + async def _think(self) -> bool: if self._rc.todo is None: self._set_state(0) - return + return True if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) else: self._rc.todo = None + return False async def _act(self) -> Message: logger.info(f"{self._setting}: ready to {self._rc.todo}") diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 217272b54..493c172ae 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-8-7, :class:`Role` + properties. @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; Change cost control from global to company level. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. """ from __future__ import annotations @@ -192,12 +193,12 @@ class Role: return self._setting.desc return PREFIX_TEMPLATE.format(**self._setting.dict()) - async def _think(self) -> None: - """思考要做什么，决定下一步的action""" + async def _think(self) -> bool: + """Consider what to do and decide on the next course of action. Return false if nothing can be done.""" if len(self._actions) == 1: # 如果只有一个动作，那就只能做这个 self._set_state(0) - return + return True prompt = self._get_prefix() prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1) @@ -207,6 +208,7 @@ class Role: logger.warning(f'Invalid answer of state, {next_state=}') next_state = "0" self._set_state(int(next_state)) + return True async def _act(self) -> Message: # prompt = self.get_prefix() diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index f29f384db..9a68fa9e0 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -4,6 +4,8 @@ @Time : 2023/7/27 @Author : mashenquan @File : teacher.py +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + """ @@ -31,16 +33,17 @@ class Teacher(Role): self._init_actions(actions) self._watch({TeachingPlanRequirement}) - async def _think(self) -> None: + async def _think(self) -> bool: """Everything will be done part by part.""" if self._rc.todo is None: self._set_state(0) - return + return True if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) else: self._rc.todo = None + return False async def _react(self) -> Message: ret = Message(content="") From 67f6fe652359f883a7e11281581260e5ffd8f21c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 23 Aug 2023 16:25:47 +0800 Subject: [PATCH 078/378] fixbug: _think return None --- metagpt/roles/teacher.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index 9a68fa9e0..d2a2198f5 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -41,9 +41,10 @@ class Teacher(Role): if self._rc.state + 1 < len(self._states): self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - return False + return True + + self._rc.todo = None + return False async def _react(self) -> Message: ret = Message(content="") From 5f16d6e8534a0b0c2316211374290f9f084ac69a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 24 Aug 2023 15:22:29 +0800 Subject: [PATCH 079/378] feat: +text summarize --- metagpt/provider/openai_api.py | 55 +++++++++++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 3baf8d932..48b7991dc 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -9,7 +9,7 @@ import asyncio import time -from typing import NamedTuple +from typing import NamedTuple, List import traceback import openai from openai.error import APIConnectionError @@ -310,3 +310,56 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @property def openai_api_version(self): return self._options.get("openai_api_version") + + async def get_summary(self, text: str, max_words=20): + """Generate text summary""" + language = self._options.get("language", "English") + command = f"Translate the above content into a {language} summary of less than {max_words} words." + msg = text + "\n\n" + command + logger.info(f"summary ask:{msg}") + response = await self.aask(msg=msg, system_msgs=[]) + logger.info(f"summary rsp: {response}") + return response + + async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: + """Generate text title""" + max_response_token_count = 50 + max_token_count = max_token_count_per_ask or self._options.get("MAX_TOKENS", 1500) + text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) + + summaries = [] + for ws in text_windows: + response = await self.get_summary(ws) + summaries.append(response) + + language = self._options.get("language", "English") + command = f"Translate the above summary into a {language} title of less than {max_words} words." + summaries.append(command) + msg = "\n".join(summaries) + logger.info(f"title ask:{msg}") + response = await self.aask(msg=msg, system_msgs=[]) + logger.info(f"title rsp: {response}") + return response + + @staticmethod + def split_texts(text: str, window_size) -> List[str]: + """Splitting long text into sliding windows text""" + total_len = len(text) + if total_len <= window_size: + return [text] + + padding_size = 20 if window_size > 20 else 0 + windows = [] + idx = 0 + while idx < total_len: + data_len = window_size - padding_size + if data_len + idx > total_len: + windows.append(text[idx:]) + break + w = text[idx:data_len] + windows.append(w) + for i in range(len(windows)): + if i + 1 == len(windows): + break + windows[i] += windows[i + 1][0:padding_size] + return windows From 799dbd396eeff71e9e5a7ab30935685b2794c9a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 25 Aug 2023 21:10:14 +0800 Subject: [PATCH 080/378] feat: archive --- .well-known/skills.yaml | 17 ++++ metagpt/actions/action.py | 2 +- metagpt/actions/action_output.py | 6 +- metagpt/actions/talk_action.py | 32 +++++++ metagpt/learn/skill_loader.py | 38 ++++++++ metagpt/memory/brain_memory.py | 47 ++++++++++ metagpt/provider/openai_api.py | 2 + metagpt/roles/assistant.py | 143 +++++++++++++++++++++++++++++++ metagpt/roles/role.py | 6 ++ metagpt/schema.py | 3 +- 10 files changed, 291 insertions(+), 5 deletions(-) create mode 100644 .well-known/skills.yaml create mode 100644 metagpt/actions/talk_action.py create mode 100644 metagpt/learn/skill_loader.py create mode 100644 metagpt/memory/brain_memory.py create mode 100644 metagpt/roles/assistant.py diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml new file mode 100644 index 000000000..5ccb8094b --- /dev/null +++ b/.well-known/skills.yaml @@ -0,0 +1,17 @@ +entities: + Assistant: + skills: + - name: text_to_speech + description: Text-to-speech + requisite: + - AZURE_TTS_SUBSCRIPTION_KEY + - AZURE_TTS_REGION + - name: text_to_image + description: Create a drawing based on the text. + requisite: + - OPENAI_API_KEY + - METAGPT_TEXT_TO_IMAGE_MODEL + - name: text_to_embedding + description: Convert the text into embeddings. + requisite: + - OPENAI_API_KEY diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 899c2515c..86a6664ba 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -62,6 +62,6 @@ class Action(ABC): instruct_content = output_class(**parsed_data) return ActionOutput(content, instruct_content) - async def run(self, *args, **kwargs): + async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" raise NotImplementedError("The run method should be implemented in a subclass.") diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index c0b88dcf9..6c812e7fe 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -6,16 +6,16 @@ @File : action_output """ -from typing import Dict, Type +from typing import Dict, Type, Optional from pydantic import BaseModel, create_model, root_validator, validator class ActionOutput: content: str - instruct_content: BaseModel + instruct_content: Optional[BaseModel] = None - def __init__(self, content: str, instruct_content: BaseModel): + def __init__(self, content: str, instruct_content: BaseModel=None): self.content = content self.instruct_content = instruct_content diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py new file mode 100644 index 000000000..4275a1b9e --- /dev/null +++ b/metagpt/actions/talk_action.py @@ -0,0 +1,32 @@ +from metagpt.actions import Action, ActionOutput +from metagpt.logs import logger + + + +class TalkAction(Action): + def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None): + context = context or {} + context["talk"] = talk + context["history_summery"] = history_summary + super(TalkAction, self).__init__(options=options, name=name, context=context, llm=llm) + self._talk = talk + self._history_summary = history_summary + self._rsp = None + + @property + def prompt(self): + prompt = f"{self._history_summary}\n\n" + if self._history_summary != "": + prompt += "According to the historical conversation above, " + language = self.options.get("language", "Chinese") + prompt += f"Answer in {language}:\n {self._talk}" + return prompt + + async def run(self, *args, **kwargs) -> ActionOutput: + prompt = self.prompt + logger.info(prompt) + rsp = await self.llm.aask(msg=prompt, system_msgs=[]) + logger.info(rsp) + self._rsp = ActionOutput(content=rsp) + return self._rsp + diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py new file mode 100644 index 000000000..eeca12871 --- /dev/null +++ b/metagpt/learn/skill_loader.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import List, Dict + +import yaml +from pydantic import BaseModel + + +class Skill(BaseModel): + name: str + description: str + requisite: List[str] + + +class EntitySkills(BaseModel): + skills: List[Skill] + + +class SkillsDeclaration(BaseModel): + entities: Dict[str, EntitySkills] + + +class SkillLoader: + def __init__(self): + skill_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" + with open(str(skill_file_name), 'r') as file: + skills = yaml.safe_load(file) + self._skills = SkillsDeclaration(**skills) + + def get_skill_list(self, entity_name: str = "Assistant"): + if not self._skills or entity_name not in self._skills.entities: + return {} + entity_skills = self._skills.entities.get(entity_name) + + description_to_name_mappings = {} + for s in entity_skills.skills: + description_to_name_mappings[s.description] = s.name + + return description_to_name_mappings diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py new file mode 100644 index 000000000..97319859a --- /dev/null +++ b/metagpt/memory/brain_memory.py @@ -0,0 +1,47 @@ +from enum import Enum +from typing import List + +import pydantic + +from metagpt import Message + +class MessageType(Enum): + Talk = "TALK" + Solution = "SOLUTION" + Problem = "PROBLEM" + Skill = "SKILL" + Answer = "ANSWER" + + +class BrainMemory(pydantic.BaseModel): + history: List[Message] = [] + stack: List[Message] = [] + solution: List[Message] = [] + + + def add_talk(self, msg: Message): + msg.add_tag(MessageType.Talk.value) + self.history.append(msg) + + def add_answer(self, msg: Message): + msg.add_tag(MessageType.Answer.value) + self.history.append(msg) + + @property + def history_text(self): + if len(self.history) == 0: + return "" + texts = [m.content for m in self.history[:-1]] + return "\n".join(texts) + + def move_to_solution(self): + while len(self.history) > 1: + msg = self.history.pop() + self.solution.append(msg) + + @property + def last_talk(self): + if len(self.history) == 0 or not self.history[-1].is_contain_tags([MessageType.Talk.value]): + return "" + return self.history[-1].content + diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 48b7991dc..06a3154e8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -313,6 +313,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_summary(self, text: str, max_words=20): """Generate text summary""" + if len(text) < max_words: + return text language = self._options.get("language", "English") command = f"Translate the above content into a {language} summary of less than {max_words} words." msg = text + "\n\n" + command diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py new file mode 100644 index 000000000..fde011892 --- /dev/null +++ b/metagpt/roles/assistant.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/7 +@Author : mashenquan +@File : fork_meta_role.py +@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the + ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to + make these symbols configurable and standardized, making the process of building flows more convenient. + For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` + This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a + configuration file. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. + +""" +import asyncio +import re + +from metagpt.actions import ActionOutput +from metagpt.actions.talk_action import TalkAction +from metagpt.config import Config +from metagpt.learn.skill_loader import SkillLoader +from metagpt.logs import logger +from metagpt.memory.brain_memory import BrainMemory, MessageType +from metagpt.provider.openai_api import CostManager +from metagpt.roles import Role +from metagpt.schema import Message + +DEFAULT_MAX_TOKENS = 1500 +COMMAND_TOKENS = 500 + + +class Assistant(Role): + """解决通用问题的助手""" + + def __init__(self, options, cost_manager, name="Lily", profile="An assistant", goal="Help to solve problem", + constraints="Talk in {language}", desc="", *args, **kwargs): + super(Assistant, self).__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, + goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + self.memory = BrainMemory() + self.skills = SkillLoader() + + async def think(self) -> bool: + """Everything will be done part by part.""" + if self.memory.history_text != "": + self._refine_memory() + + + prompt = "" + history_text = self.memory.history_text + history_summary = "" + if history_text != "": + max_tokens = self.options.get("MAX_TOKENS", DEFAULT_MAX_TOKENS) + history_summary = await self._llm.get_summary(history_text, max_tokens - COMMAND_TOKENS) + prompt += history_summary + "\n\n" + prompt += "Analyze the conversation history above, in conjunction with the current sentence: \n{self.memory.last_talk}\n\n" + else: + prompt += f"Refer to this sentence:\n {self.memory.last_talk}\n" + skills = self.skills.get_skill_list() + for desc, name in skills.items(): + prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" + if history_text != "": + prompt += "If the last sentence is not related to the conversation history above, return `[SOLUTION]: {title of the history conversation}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" + prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" + prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" + prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + logger.info(prompt) + rsp = await self._llm.aask(prompt, []) + logger.info(rsp) + return await self._plan(rsp, history_summary=history_summary) + + async def act(self) -> ActionOutput: + result = await self._rc.todo.run(**self._options) + if not result: + return None + if isinstance(result, str): + msg = Message(content=result) + output = ActionOutput(content=result) + else: + msg = Message(content=result.content, instruct_content=result.instruct_content, + cause_by=type(self._rc.todo)) + output = result + self.memory.add_answer(msg) + return output + + async def talk(self, text): + self.memory.add_talk(Message(content=text, tags=set([MessageType.Talk.value]))) + + async def _plan(self, rsp, **kwargs) -> bool: + skill, text = Assistant.extract_info(rsp) + handlers = { + MessageType.Talk.value: self.talk_handler, + MessageType.Problem.value: self.problem_handler, + MessageType.Solution.value: self.solution_handler, + MessageType.Skill.value: self.skill_handler, + } + handler = handlers.get(skill, self.talk_handler) + return await handler(text, **kwargs) + + @staticmethod + def extract_info(input_string): + pattern = r'\[([A-Z]+)\]:\s*(.+)' + match = re.match(pattern, input_string) + if match: + return match.group(1), match.group(2) + else: + return None, input_string + + async def problem_handler(self, text, **kwargs) -> bool: + action = TalkAction(options=self.options, talk=text, llm=self._llm, **kwargs) + self.add_to_do(action) + return True + + async def solution_handler(self, text, **kwargs) -> bool: + self.memory.move_to_solution() # 问题解决后及时清空内存 + action = TalkAction(options=self.options, talk=text, history_summary="", **kwargs) + self.add_to_do(action) + + async def skill_handler(self, text, **kwargs) -> bool: + pass + + async def _refine_memory(self): + + +async def main(): + options = Config().runtime_options + cost_manager = CostManager(**options) + topic = "dataiku vs. datarobot" + role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") + await role.talk(topic) + while True: + has_action = await role.think() + if not has_action: + break + msg = await role.act() + print(msg) + # 获取用户终端输入 + talk = input("You: ") + await role.talk(talk) + + +if __name__ == '__main__': + asyncio.run(main()) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 493c172ae..1bb73f884 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -320,3 +320,9 @@ class Role: for k, v in merged_opts.items(): value = value.replace("{" + f"{k}" + "}", str(v)) return value + + def add_action(self, act): + self._actions.append(act) + + def add_to_do(self, act): + self._rc.todo = act \ No newline at end of file diff --git a/metagpt/schema.py b/metagpt/schema.py index 4c577fd7b..e1cd011c6 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field from enum import Enum -from typing import Type, TypedDict, Set, Optional +from typing import Type, TypedDict, Set, Optional, List from pydantic import BaseModel @@ -98,6 +98,7 @@ class AIMessage(Message): super().__init__(content, 'assistant') + if __name__ == '__main__': test_content = 'test_message' msgs = [ From 1aeebc85fbba23d96bb8396775636123ac1b929b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 25 Aug 2023 21:54:28 +0800 Subject: [PATCH 081/378] feat: archive --- metagpt/provider/openai_api.py | 23 ++++++++++++ metagpt/roles/assistant.py | 64 ++++++++++++++-------------------- 2 files changed, 49 insertions(+), 38 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 06a3154e8..510041e98 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -7,6 +7,7 @@ Change cost control from global to company level. """ import asyncio +import re import time from typing import NamedTuple, List @@ -333,6 +334,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): for ws in text_windows: response = await self.get_summary(ws) summaries.append(response) + if len(summaries) == 1: + return summaries[0] language = self._options.get("language", "English") command = f"Translate the above summary into a {language} title of less than {max_words} words." @@ -343,6 +346,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.info(f"title rsp: {response}") return response + async def is_related(self, text1, text2): + command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + rsp = await self.aask(msg=command, system_msgs=[]) + result, _ = self.extract_info(rsp) + return result == "TRUE" + + async def rewrite(self, sentence: str, context: str): + command = f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + rsp = await self.aask(msg=command, system_msgs=[]) + return rsp + @staticmethod def split_texts(text: str, window_size) -> List[str]: """Splitting long text into sliding windows text""" @@ -365,3 +379,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): break windows[i] += windows[i + 1][0:padding_size] return windows + + @staticmethod + def extract_info(input_string): + pattern = r'\[([A-Z]+)\]:\s*(.+)' + match = re.match(pattern, input_string) + if match: + return match.group(1), match.group(2) + else: + return None, input_string \ No newline at end of file diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index fde011892..dfbd406bc 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -14,7 +14,7 @@ """ import asyncio -import re + from metagpt.actions import ActionOutput from metagpt.actions.talk_action import TalkAction @@ -42,32 +42,18 @@ class Assistant(Role): async def think(self) -> bool: """Everything will be done part by part.""" - if self.memory.history_text != "": - self._refine_memory() - - - prompt = "" - history_text = self.memory.history_text - history_summary = "" - if history_text != "": - max_tokens = self.options.get("MAX_TOKENS", DEFAULT_MAX_TOKENS) - history_summary = await self._llm.get_summary(history_text, max_tokens - COMMAND_TOKENS) - prompt += history_summary + "\n\n" - prompt += "Analyze the conversation history above, in conjunction with the current sentence: \n{self.memory.last_talk}\n\n" - else: - prompt += f"Refer to this sentence:\n {self.memory.last_talk}\n" + last_talk = await self.refine_memory() + prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" - if history_text != "": - prompt += "If the last sentence is not related to the conversation history above, return `[SOLUTION]: {title of the history conversation}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) - return await self._plan(rsp, history_summary=history_summary) + return await self._plan(rsp) async def act(self) -> ActionOutput: result = await self._rc.todo.run(**self._options) @@ -86,40 +72,42 @@ class Assistant(Role): async def talk(self, text): self.memory.add_talk(Message(content=text, tags=set([MessageType.Talk.value]))) - async def _plan(self, rsp, **kwargs) -> bool: - skill, text = Assistant.extract_info(rsp) + async def _plan(self, rsp: str, **kwargs) -> bool: + skill, text = Assistant.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, - MessageType.Problem.value: self.problem_handler, - MessageType.Solution.value: self.solution_handler, + MessageType.Problem.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, } handler = handlers.get(skill, self.talk_handler) return await handler(text, **kwargs) - @staticmethod - def extract_info(input_string): - pattern = r'\[([A-Z]+)\]:\s*(.+)' - match = re.match(pattern, input_string) - if match: - return match.group(1), match.group(2) - else: - return None, input_string - - async def problem_handler(self, text, **kwargs) -> bool: + async def talk_handler(self, text, **kwargs) -> bool: action = TalkAction(options=self.options, talk=text, llm=self._llm, **kwargs) self.add_to_do(action) return True - async def solution_handler(self, text, **kwargs) -> bool: - self.memory.move_to_solution() # 问题解决后及时清空内存 - action = TalkAction(options=self.options, talk=text, history_summary="", **kwargs) - self.add_to_do(action) - async def skill_handler(self, text, **kwargs) -> bool: + skill = pass - async def _refine_memory(self): + async def refine_memory(self) -> str: + history_text = self.memory.history_text + last_talk = self.memory.last_talk + if history_text == "": + return last_talk + history_summary = await self._llm.get_context_title(history_text, max_words=20) + if await self._llm.is_related(last_talk, history_summary): # 合并相关内容 + last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + return last_talk + + self.memory.move_to_solution() # 问题解决后及时清空内存 + return last_talk + + @staticmethod + def extract_info(input_string): + from metagpt.provider.openai_api import OpenAIGPTAPI + return OpenAIGPTAPI.extract_info(input_string) async def main(): From 0821e6d0996d886546d9134f7bc62f35162dddb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 10:21:49 +0800 Subject: [PATCH 082/378] feat: + RateLimitError retry --- metagpt/provider/openai_api.py | 29 ++++++++++++++++++++--------- 1 file changed, 20 insertions(+), 9 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 510041e98..e98acbd75 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -9,6 +9,7 @@ import asyncio import re import time +import random from typing import NamedTuple, List import traceback @@ -152,15 +153,25 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(self._options.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - try: - response = await openai.ChatCompletion.acreate( - **self._cons_kwargs(messages), - stream=True - ) - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e + max_try = 5 + response = None + for i in range(max_try): + try: + response = await openai.ChatCompletion.acreate( + **self._cons_kwargs(messages), + stream=True + ) + break + except openai.error.RateLimitError as e: + random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 + rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 + logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") + await asyncio.sleep(rounded_time) + continue + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e # create variables to collect the stream of chunks collected_chunks = [] From 4fe3d6e8790f17d01ab059b5fb5d01d02328540a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 16:52:21 +0800 Subject: [PATCH 083/378] fixbug: unit test --- metagpt/actions/skill_action.py | 0 metagpt/tools/metagpt_text_to_image.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 metagpt/actions/skill_action.py diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py new file mode 100644 index 000000000..e69de29bb diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 393215df0..674ff283a 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -105,7 +105,7 @@ def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): if __name__ == "__main__": initialize_environment() - v = oas3_metagpt_text_2_image("Panda emoji") + v = oas3_metagpt_text_to_image("Panda emoji") data = base64.b64decode(v) with open("tmp.png", mode="wb") as writer: writer.write(data) From 2c593bedea549e5068e1c92ff264908d93add0f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 16:59:12 +0800 Subject: [PATCH 084/378] feat: +common talk role --- .well-known/skills.yaml | 34 ++++++++++-- metagpt/actions/skill_action.py | 88 ++++++++++++++++++++++++++++++ metagpt/actions/talk_action.py | 2 +- metagpt/learn/__init__.py | 8 +++ metagpt/learn/skill_loader.py | 33 +++++++++-- metagpt/learn/text_to_embedding.py | 2 +- metagpt/learn/text_to_image.py | 12 +++- metagpt/learn/text_to_speech.py | 6 +- metagpt/memory/brain_memory.py | 12 +++- metagpt/provider/openai_api.py | 63 ++++++++++++++------- metagpt/roles/assistant.py | 34 +++++++++--- metagpt/roles/role.py | 10 +++- metagpt/schema.py | 3 + 13 files changed, 261 insertions(+), 46 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 5ccb8094b..7a035910c 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -3,15 +3,41 @@ entities: skills: - name: text_to_speech description: Text-to-speech + id: text_to_speech.text_to_speech requisite: - AZURE_TTS_SUBSCRIPTION_KEY - AZURE_TTS_REGION + arguments: + text: 'The text used for voice conversion. Required.' + lang: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States). The optional parameter are "English", "Chinese". Default value: "Chinese".' + voice: 'Default value: "zh-CN-XiaomoNeural".' + style: 'Speaking style to express different emotions like cheerfulness, empathy, and calm. The optional parameter values are "affectionate", "angry", "calm", "cheerful", "depressed", "disgruntled", "embarrassed", "envious", "fearful", "gentle", "sad", "serious". Default value: "affectionate".' + role: 'With roles, the same voice can act as a different age and gender. The optional parameter values are "Girl", "Boy", "OlderAdultFemale", "OlderAdultMale", "SeniorFemale", "SeniorMale", "YoungAdultFemale", "YoungAdultMale". Default value: "Girl".' + examples: + - ask: 'A girl says "hello world"' + answer: 'text_to_speech(text="hello world", role="Girl")' + - ask: 'A boy affectionate says "hello world"' + answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' + - ask: 'A boy says "你好"' + answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + returns: + type: string + format: base64 + - name: text_to_image description: Create a drawing based on the text. + id: text_to_image.text_to_image requisite: - OPENAI_API_KEY - METAGPT_TEXT_TO_IMAGE_MODEL - - name: text_to_embedding - description: Convert the text into embeddings. - requisite: - - OPENAI_API_KEY + arguments: + text: 'The text used for image conversion. Required.' + size_type: 'Default value: "512x512".' + examples: + - ask: 'Draw a girl' + answer: 'text_to_image(text="Draw a girl", size_type="512x512")' + - ask: 'Draw an apple' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + returns: + type: string + format: base64 diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index e69de29bb..8cc7b6c42 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -0,0 +1,88 @@ +import ast +import importlib + +from metagpt.actions import Action, ActionOutput +from metagpt.learn.skill_loader import Skill +from metagpt.logs import logger + + +class ArgumentsParingAction(Action): + def __init__(self, options, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): + super(ArgumentsParingAction, self).__init__(options=options, name='', context=context, llm=llm) + self.skill = skill + self.ask = last_talk + self.rsp = None + self.args = None + + @property + def prompt(self): + prompt = f"{self.skill.name} function parameters description:\n" + for k, v in self.skill.arguments.items(): + prompt += f"parameter `{k}`: {v}\n" + prompt += "\n" + prompt += "Examples:\n" + for e in self.skill.examples: + prompt += f"If want you to do `{e.ask}`, return `{e.answer}` brief and clear.\n" + prompt += f"\nNow I want you to do `{self.ask}`, return in examples format above, brief and clear." + return prompt + + async def run(self, *args, **kwargs) -> ActionOutput: + prompt = self.prompt + logger.info(prompt) + rsp = await self.llm.aask(msg=prompt, system_msgs=[]) + logger.info(rsp) + self.args = ArgumentsParingAction.parse_arguments(skill_name=self.skill.name, txt=rsp) + self.rsp = ActionOutput(content=rsp) + return self.rsp + + @staticmethod + def parse_arguments(skill_name, txt) -> dict: + prefix = skill_name + "(" + if prefix not in txt: + logger.error(f"{skill_name} not in {txt}") + return None + if ")" not in txt: + logger.error(f"')' not in {txt}") + return None + begin_ix = txt.find(prefix) + end_ix = txt.rfind(")") + args_txt = txt[begin_ix + len(prefix): end_ix] + logger.info(args_txt) + fake_expression = f"dict({args_txt})" + parsed_expression = ast.parse(fake_expression, mode='eval') + args = {} + for keyword in parsed_expression.body.keywords: + key = keyword.arg + value = ast.literal_eval(keyword.value) + args[key] = value + return args + + +class SkillAction(Action): + def __init__(self, options, skill: Skill, args: dict, context=None, llm=None, **kwargs): + super(SkillAction, self).__init__(options=options, name='', context=context, llm=llm) + self._skill = skill + self._args = args + self.rsp = None + + async def run(self, *args, **kwargs) -> str | ActionOutput | None: + """Run action""" + self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **self.options) + return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) + + @staticmethod + def find_and_call_function(function_name, args, **kwargs): + try: + module = importlib.import_module("metagpt.learn") + function = getattr(module, function_name) + # 调用函数并返回结果 + result = function(**args, **kwargs) + return result + except (ModuleNotFoundError, AttributeError): + logger.error(f"{function_name} not found") + return None + + +if __name__ == '__main__': + ArgumentsParingAction.parse_arguments(skill_name="text_to_image", + txt='`text_to_image(text="Draw an apple", size_type="512x512")`') diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 4275a1b9e..5485456c5 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -4,7 +4,7 @@ from metagpt.logs import logger class TalkAction(Action): - def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None): + def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary diff --git a/metagpt/learn/__init__.py b/metagpt/learn/__init__.py index 28b8739c3..c8270dbfb 100644 --- a/metagpt/learn/__init__.py +++ b/metagpt/learn/__init__.py @@ -5,3 +5,11 @@ @Author : alexanderwu @File : __init__.py """ + +from metagpt.learn.text_to_image import text_to_image +from metagpt.learn.text_to_speech import text_to_speech + +__all__ = [ + "text_to_image", + "text_to_speech", +] \ No newline at end of file diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index eeca12871..46ead728d 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -1,14 +1,26 @@ from pathlib import Path -from typing import List, Dict +from typing import List, Dict, Optional import yaml from pydantic import BaseModel +class Example(BaseModel): + ask: str + answer: str + +class Returns(BaseModel): + type: str + format: Optional[str] = None + class Skill(BaseModel): name: str description: str + id: str requisite: List[str] + arguments: Dict + examples: List[Example] + returns: Returns class EntitySkills(BaseModel): @@ -26,13 +38,26 @@ class SkillLoader: skills = yaml.safe_load(file) self._skills = SkillsDeclaration(**skills) - def get_skill_list(self, entity_name: str = "Assistant"): - if not self._skills or entity_name not in self._skills.entities: + def get_skill_list(self, entity_name: str = "Assistant") -> Dict: + entity_skills = self.get_entity(entity_name) + if not entity_skills: return {} - entity_skills = self._skills.entities.get(entity_name) description_to_name_mappings = {} for s in entity_skills.skills: description_to_name_mappings[s.description] = s.name return description_to_name_mappings + + def get_skill(self, name, entity_name: str = "Assistant") -> Skill: + entity = self.get_entity(entity_name) + if not entity: + return None + for sk in entity.skills: + if sk.name == name: + return sk + + def get_entity(self, name) -> EntitySkills: + if not self._skills: + return None + return self._skills.entities.get(name) \ No newline at end of file diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 38fd7c0cb..6d0cefcdb 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -16,7 +16,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to Embedding", description="Convert the text into embeddings.", requisite="`OPENAI_API_KEY`") -def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): +def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): """Text to embedding :param text: The text used for embedding. diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index d123e116a..2f946e239 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -17,7 +17,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to image", description="Create a drawing based on the text.", requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") -def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url=""): +def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): """Text to image :param text: The text used for image conversion. @@ -27,8 +27,14 @@ def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url :return: The image data is returned in Base64 encoding. """ initialize_environment() + image_declaration = "data:image/png;base64," if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL") or model_url: - return oas3_metagpt_text_to_image(text, size_type, model_url) + data = oas3_metagpt_text_to_image(text, size_type, model_url) + return image_declaration + data if data else "" if os.environ.get("OPENAI_API_KEY") or openai_api_key: - return oas3_openai_text_to_image(text, size_type, openai_api_key) + data = oas3_openai_text_to_image(text, size_type, openai_api_key) + return image_declaration + data if data else "" + raise EnvironmentError + + diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 5631ef45e..90dd878a1 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -17,7 +17,7 @@ from metagpt.utils.common import initialize_environment description="Text-to-speech", requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", - subscription_key="", region=""): + subscription_key="", region="", **kwargs): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -32,8 +32,10 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect """ initialize_environment() + audio_declaration = "data:audio/wav;base64," if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ (subscription_key and region): - return oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + data = oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + return audio_declaration + data if data else data raise EnvironmentError diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 97319859a..68e930144 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -35,9 +35,15 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self): - while len(self.history) > 1: - msg = self.history.pop() - self.solution.append(msg) + if len(self.history) < 2: + return + msgs = self.history[:-1] + self.solution.extend(msgs) + if not self.history[-1].is_contain(MessageType.Talk.value): + self.solution.append(self.history[-1]) + self.history = [] + else: + self.history = self.history[-1:] @property def last_talk(self): diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e98acbd75..27f22e491 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -153,26 +153,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(self._options.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - max_try = 5 - response = None - for i in range(max_try): - try: - response = await openai.ChatCompletion.acreate( + response = await self.async_retry_call(openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True ) - break - except openai.error.RateLimitError as e: - random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 - rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 - logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") - await asyncio.sleep(rounded_time) - continue - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e - # create variables to collect the stream of chunks collected_chunks = [] collected_messages = [] @@ -213,12 +197,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: - rsp = await self.llm.ChatCompletion.acreate(**self._cons_kwargs(messages)) + rsp = await self.async_retry_call(self.llm.ChatCompletion.acreate, **self._cons_kwargs(messages)) self._update_costs(rsp.get("usage")) return rsp def _chat_completion(self, messages: list[dict]) -> dict: - rsp = self.llm.ChatCompletion.create(**self._cons_kwargs(messages)) + rsp = self.retry_call(self.llm.ChatCompletion.create, **self._cons_kwargs(messages)) self._update_costs(rsp) return rsp @@ -398,4 +382,43 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if match: return match.group(1), match.group(2) else: - return None, input_string \ No newline at end of file + return None, input_string + + @staticmethod + async def async_retry_call(func, *args, **kwargs): + for i in range(OpenAIGPTAPI.MAX_TRY): + try: + rsp = await func(*args, **kwargs) + return rsp + except openai.error.RateLimitError as e: + random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 + rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 + logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") + await asyncio.sleep(rounded_time) + continue + except openai.error.APIConnectionError as e: + logger.warning(f"Exception:{e}") + continue + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e + + @staticmethod + def retry_call(func, *args, **kwargs): + for i in range(OpenAIGPTAPI.MAX_TRY): + try: + rsp = func(*args, **kwargs) + return rsp + except openai.error.RateLimitError as e: + logger.warning(f"Exception:{e}") + continue + except openai.error.APIConnectionError as e: + logger.warning(f"Exception:{e}") + continue + except Exception as e: + error_str = traceback.format_exc() + logger.error(f"Exception:{e}, stack:{error_str}") + raise e + + MAX_TRY = 5 diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index dfbd406bc..032d73ca5 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -15,8 +15,8 @@ """ import asyncio - from metagpt.actions import ActionOutput +from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction from metagpt.config import Config from metagpt.learn.skill_loader import SkillLoader @@ -53,7 +53,7 @@ class Assistant(Role): logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) - return await self._plan(rsp) + return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: result = await self._rc.todo.run(**self._options) @@ -88,8 +88,18 @@ class Assistant(Role): return True async def skill_handler(self, text, **kwargs) -> bool: - skill = - pass + last_talk = kwargs.get("last_talk") + skill = self.skills.get_skill(text) + logger.info(f"skill not found: {text}") + if not skill: + return await self.talk_handler(text=last_talk, **kwargs) + action = ArgumentsParingAction(options=self.options, skill=skill, llm=self._llm, **kwargs) + await action.run(**kwargs) + if action.args is None: + return await self.talk_handler(text=last_talk, **kwargs) + action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm) + self.add_to_do(action) + return True async def refine_memory(self) -> str: history_text = self.memory.history_text @@ -97,7 +107,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_context_title(history_text, max_words=20) - if await self._llm.is_related(last_talk, history_summary): # 合并相关内容 + if last_talk and await self._llm.is_related(last_talk, history_summary): # 合并相关内容 last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk @@ -109,11 +119,20 @@ class Assistant(Role): from metagpt.provider.openai_api import OpenAIGPTAPI return OpenAIGPTAPI.extract_info(input_string) + def get_memory(self) -> str: + return self.memory.json() + + def load_memory(self, jsn): + try: + self.memory = BrainMemory(**jsn) + except Exception as e: + logger.exception(f"load error:{e}, data:{jsn}") + async def main(): options = Config().runtime_options cost_manager = CostManager(**options) - topic = "dataiku vs. datarobot" + topic = "draw an apple" role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") await role.talk(topic) while True: @@ -121,8 +140,9 @@ async def main(): if not has_action: break msg = await role.act() - print(msg) + logger.info(msg) # 获取用户终端输入 + logger.info("Enter prompt") talk = input("You: ") await role.talk(talk) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 1bb73f884..47f494c69 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -325,4 +325,12 @@ class Role: self._actions.append(act) def add_to_do(self, act): - self._rc.todo = act \ No newline at end of file + self._rc.todo = act + + async def think(self) -> bool: + return await self._think() + + async def act(self) -> ActionOutput: + msg = await self._act() + return ActionOutput(content=msg.content, + instruct_content=msg.instruct_content) diff --git a/metagpt/schema.py b/metagpt/schema.py index e1cd011c6..909313886 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -67,6 +67,9 @@ class Message: intersection = set(tags) & self.tags return len(intersection) > 0 + def is_contain(self, tag): + return self.is_contain_tags([tag]) + @dataclass class UserMessage(Message): From 644286959152f933bc84815704194080ad86e5e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 17:11:33 +0800 Subject: [PATCH 085/378] feat: +common talk role --- metagpt/roles/assistant.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 032d73ca5..f75c05695 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -43,6 +43,8 @@ class Assistant(Role): async def think(self) -> bool: """Everything will be done part by part.""" last_talk = await self.refine_memory() + if not last_talk: + return False prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): From 6e459da875896e094826841814714f3fdf9b1911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 17:20:21 +0800 Subject: [PATCH 086/378] feat: +Exceeds the maximum retries exception --- metagpt/provider/openai_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 27f22e491..4fab92fb3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -403,6 +403,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") raise e + raise openai.error.OpenAIError("Exceeds the maximum retries") @staticmethod def retry_call(func, *args, **kwargs): @@ -420,5 +421,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") raise e + raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 From 5dc352bf2fd102732a525f7d1020c91889a5f0be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 19:18:23 +0800 Subject: [PATCH 087/378] feat: fix requirements-test.txt --- requirements-test.txt | 40 +--------------------------------------- requirements.txt | 2 +- 2 files changed, 2 insertions(+), 40 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 7c03dddd9..0a34c35ea 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -1,41 +1,3 @@ -aiohttp==3.8.4 -azure-cognitiveservices-speech==1.30.0 -channels==4.0.0 -chromadb==0.3.22 -# Django==4.1.5 -# docx==0.2.4 -duckduckgo_search==2.9.4 -#faiss==1.5.3 -faiss_cpu==1.7.4 -fire==0.4.0 -# godot==0.1.1 -# google_api_python_client==2.93.0 -langchain==0.0.231 -loguru==0.6.0 -meilisearch==0.21.0 -numpy==1.24.3 -openai==0.27.8 -openpyxl -pandas==1.4.1 -pydantic==1.10.7 -#pygame==2.1.3 -pymilvus==2.2.8 -pytest==7.2.2 -python_docx==0.8.11 -PyYAML==6.0 -# sentence_transformers==2.2.2 -setuptools==65.6.3 -tenacity==8.2.2 -tiktoken==0.3.3 -tqdm==4.64.0 -#unstructured[local-inference] -playwright -selenium>4 -webdriver_manager<3.9 -anthropic==0.3.6 -typing-inspect==0.8.0 -typing_extensions==4.5.0 -bs4 -aiofiles +-r requirements.txt pytest pytest-asyncio \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4bfab1f3b..70f2a3809 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,4 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 - +azure-cognitiveservices-speech==1.30.0 From 2c83921aee8231696947c6dacfc66f340a739648 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 19:54:49 +0800 Subject: [PATCH 088/378] feat: +brain memory --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index f75c05695..e02005f31 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -28,6 +28,7 @@ from metagpt.schema import Message DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 +BRAIN_MEMORY = "BRAIN_MEMORY" class Assistant(Role): @@ -37,7 +38,8 @@ class Assistant(Role): constraints="Talk in {language}", desc="", *args, **kwargs): super(Assistant, self).__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) - self.memory = BrainMemory() + brain_memory = options.get(BRAIN_MEMORY) + self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() self.skills = SkillLoader() async def think(self) -> bool: From 6e10cbb73bd19b01cf70146c06fa63effc3db4d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 20:18:47 +0800 Subject: [PATCH 089/378] feat: +knowledge --- metagpt/actions/talk_action.py | 7 +++++-- metagpt/memory/brain_memory.py | 5 +++++ metagpt/roles/assistant.py | 5 +++-- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 5485456c5..dab4873fb 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -4,18 +4,21 @@ from metagpt.logs import logger class TalkAction(Action): - def __init__(self, options, name: str = '', talk='', history_summary='', context=None, llm=None, **kwargs): + def __init__(self, options, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary + context["knowledge"] = knowledge super(TalkAction, self).__init__(options=options, name=name, context=context, llm=llm) self._talk = talk self._history_summary = history_summary + self._knowledge = knowledge self._rsp = None @property def prompt(self): - prompt = f"{self._history_summary}\n\n" + prompt = f"{self._knowledge}\n\n" + prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " language = self.options.get("language", "Chinese") diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 68e930144..422c096f3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -17,6 +17,7 @@ class BrainMemory(pydantic.BaseModel): history: List[Message] = [] stack: List[Message] = [] solution: List[Message] = [] + knowledge: List[Message] = [] def add_talk(self, msg: Message): @@ -27,6 +28,10 @@ class BrainMemory(pydantic.BaseModel): msg.add_tag(MessageType.Answer.value) self.history.append(msg) + def get_knowledge(self) -> str: + texts = [k.content for k in self.knowledge] + return "\n".join(texts) + @property def history_text(self): if len(self.history) == 0: diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index e02005f31..c001d69f0 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -87,7 +87,8 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(options=self.options, talk=text, llm=self._llm, **kwargs) + action = TalkAction(options=self.options, talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, + **kwargs) self.add_to_do(action) return True @@ -136,7 +137,7 @@ class Assistant(Role): async def main(): options = Config().runtime_options cost_manager = CostManager(**options) - topic = "draw an apple" + topic = "what's apple" role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") await role.talk(topic) while True: From d35dc8bfefd250c57a306fba8bce725bb4578aa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 20:23:50 +0800 Subject: [PATCH 090/378] feat: +knowledge --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index dab4873fb..b1410d34f 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -17,7 +17,7 @@ class TalkAction(Action): @property def prompt(self): - prompt = f"{self._knowledge}\n\n" + prompt = f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " From 9ff489b6c68f7e668496dede44d9f6c1bff86cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 20:24:57 +0800 Subject: [PATCH 091/378] feat: +knowledge --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index b1410d34f..5692cf4f4 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -17,7 +17,7 @@ class TalkAction(Action): @property def prompt(self): - prompt = f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt = f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " From cc89f3b7263e24a445f42fec92282480029a1660 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 21:55:34 +0800 Subject: [PATCH 092/378] feat: revert --- metagpt/memory/memory.py | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index 1a8003fba..a96aaf1be 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -4,8 +4,6 @@ @Time : 2023/5/20 12:15 @Author : alexanderwu @File : memory.py -@Modified By: mashenquan, 2023-8-7. Modified get_by_actions() to support for dynamically generated Action classes - at runtime. """ from collections import defaultdict from typing import Iterable, Type @@ -82,20 +80,8 @@ class Memory: def get_by_actions(self, actions: Iterable[Type[Action]]) -> list[Message]: """Return all messages triggered by specified Actions""" rsp = [] - # Using the `type(obj).__name__` approach to support the runtime creation of requirement classes. - # See `MetaAction.get_action_type()` for more. - class_names = {type(k).__name__: k for k in self.index.keys()} for action in actions: - if type(action).__name__ not in class_names: + if action not in self.index: continue - key = class_names[type(action).__name__] - rsp += self.index[key] + rsp += self.index[action] return rsp - - def get_by_tags(self, tags: list) -> list[Message]: - """Return messages with specified tags""" - result = [] - for m in self.storage: - if m.is_contain_tags(tags): - result.append(m) - return result From 2574ecaecfb4054da2e42b81573f5e52ba8ac73f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 22:08:45 +0800 Subject: [PATCH 093/378] =?UTF-8?q?feat:=20=E5=88=A0=E6=8E=89meta=20role?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fork_meta_role_write_teaching_plan.py | 126 ----------------- metagpt/provider/openai_api.py | 6 - metagpt/roles/fork_meta_role.py | 133 ------------------ metagpt/roles/uml_meta_role_factory.py | 43 ------ metagpt/roles/uml_meta_role_options.py | 69 --------- tests/metagpt/actions/test_meta_action.py | 51 ------- tests/metagpt/roles/test_fork_meta_role.py | 94 ------------- .../roles/test_uml_meta_role_factory.py | 61 -------- .../roles/test_uml_meta_role_options.py | 40 ------ 9 files changed, 623 deletions(-) delete mode 100644 examples/fork_meta_role_write_teaching_plan.py delete mode 100644 metagpt/roles/fork_meta_role.py delete mode 100644 metagpt/roles/uml_meta_role_factory.py delete mode 100644 metagpt/roles/uml_meta_role_options.py delete mode 100644 tests/metagpt/actions/test_meta_action.py delete mode 100644 tests/metagpt/roles/test_fork_meta_role.py delete mode 100644 tests/metagpt/roles/test_uml_meta_role_factory.py delete mode 100644 tests/metagpt/roles/test_uml_meta_role_options.py diff --git a/examples/fork_meta_role_write_teaching_plan.py b/examples/fork_meta_role_write_teaching_plan.py deleted file mode 100644 index e529a9b46..000000000 --- a/examples/fork_meta_role_write_teaching_plan.py +++ /dev/null @@ -1,126 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : fork_meta_role.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to possess the - ability to construct flows freely by concatenating symbols. Simultaneously, I am also striving to make - these symbols configurable and standardized, making the process of building flow structures more - convenient. This is a fork meta-role demo that implements the functionality of - `examples/write_teaching_plan.py`. -""" - -import asyncio -from pathlib import Path -import sys - -sys.path.append(str(Path(__file__).resolve().parent.parent)) -import aiofiles -import fire -import yaml - -from metagpt.actions.meta_action import MetaAction -from metagpt.logs import logger -from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory -from metagpt.roles.uml_meta_role_options import ProjectConfig -from metagpt.software_company import SoftwareCompany - - -async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, *args, **kwargs): - """Run a startup. Be a teacher in education industry.""" - - demo_lesson = """ - UNIT 1 Making New Friends - TOPIC 1 Welcome to China! - Section A - - 1a Listen and number the following names. - Jane Mari Kangkang Michael - Look, listen and understand. Then practice the conversation. - Work in groups. Introduce yourself using - I ’m ... Then practice 1a - with your own hometown or the following places. - - 1b Listen and number the following names - Jane Michael Maria Kangkang - 1c Work in groups. Introduce yourself using I ’m ... Then practice 1a with your own hometown or the following places. - China the USA the UK Hong Kong Beijing - - 2a Look, listen and understand. Then practice the conversation - Hello! - Hello! - Hello! - Hello! Are you Maria? - No, I’m not. I’m Jane. - Oh, nice to meet you, Jane - Nice to meet you, too. - Hi, Maria! - Hi, Kangkang! - Welcome to China! - Thanks. - - 2b Work in groups. Make up a conversation with your own name and the - following structures. - A: Hello! / Good morning! / Hi! I’m ... Are you ... ? - B: ... - - 3a Listen, say and trace - Aa Bb Cc Dd Ee Ff Gg - - 3b Listen and number the following letters. Then circle the letters with the same sound as Bb. - Aa Bb Cc Dd Ee Ff Gg - - 3c Match the big letters with the small ones. Then write them on the lines. - """ - - lesson = "" - if lesson_file and Path(lesson_file).exists(): - async with aiofiles.open(lesson_file, mode="r", encoding="utf-8") as reader: - lesson = await reader.read() - logger.info(f"Course content: {lesson}") - if not lesson: - logger.info("No course content provided, using the demo course.") - lesson = demo_lesson - - yaml_filename = kwargs["config"] - kwargs["lesson"] = lesson - - with open(yaml_filename, "r") as reader: - configs = yaml.safe_load(reader) - - startup_config = ProjectConfig(**configs) - company = SoftwareCompany() - roles = UMLMetaRoleFactory.create_roles(role_configs=startup_config.roles, - options=company.options, - cost_manager=company.cost_manager, - **kwargs) - company.hire(roles) - company.invest(startup_config.startup.investment) - company.start_project(lesson, role=startup_config.startup.role, - cause_by=MetaAction.get_action_type(startup_config.startup.requirement)) - await company.run(n_round=startup_config.startup.n_round) - - -def main(idea: str, investment: float = 3.0, n_round: int = 5, *args, **kwargs): - """ - We are a software startup comprised of AI. By investing in us, you are empowering a future filled with limitless possibilities. - :param idea: lesson filename. - :param investment: As an investor, you have the opportunity to contribute a certain dollar amount to this AI company. - :param n_round: Reserved. - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - :return: - """ - asyncio.run(startup(idea, investment, n_round, *args, **kwargs)) - - -if __name__ == '__main__': - """ - Formats: - ``` - python write_teaching_plan.py lesson_filename --teaching_language= --language= - ``` - If `lesson_filename` is not available, a demo lesson content will be used. - """ - fire.Fire(main) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 4fab92fb3..098388a7c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -396,9 +396,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") await asyncio.sleep(rounded_time) continue - except openai.error.APIConnectionError as e: - logger.warning(f"Exception:{e}") - continue except Exception as e: error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") @@ -414,9 +411,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): except openai.error.RateLimitError as e: logger.warning(f"Exception:{e}") continue - except openai.error.APIConnectionError as e: - logger.warning(f"Exception:{e}") - continue except Exception as e: error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") diff --git a/metagpt/roles/fork_meta_role.py b/metagpt/roles/fork_meta_role.py deleted file mode 100644 index 57d467080..000000000 --- a/metagpt/roles/fork_meta_role.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : fork_meta_role.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` - This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a - configuration file. -@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. - -""" - -import re - -import aiofiles - -from metagpt.actions.meta_action import MetaAction -from metagpt.const import WORKSPACE_ROOT -from metagpt.logs import logger -from metagpt.roles import Role -from metagpt.roles.uml_meta_role_options import MetaActionOptions, UMLMetaRoleOptions -from metagpt.schema import Message - - -class ForkMetaRole(Role): - """A `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file""" - def __init__(self, options, cost_manager, role_options, **kwargs): - """Initialize a `fork` style meta role - - :param options: System configuration - :param cost_manager: Cost manager - :param role_options: pattern yaml file data - :param args: Parameters passed in format: `python your_script.py arg1 arg2 arg3` - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - """ - opts = UMLMetaRoleOptions(**role_options) - global_variables = { - "name": Role.format_value(opts.name, kwargs), - "profile": Role.format_value(opts.profile, kwargs), - "goal": Role.format_value(opts.goal, kwargs), - "constraints": Role.format_value(opts.constraints, kwargs), - "desc": Role.format_value(opts.desc, kwargs), - "role": Role.format_value(opts.role, kwargs) - } - for k, v in kwargs.items(): - if k not in global_variables: - global_variables[k] = v - - super(ForkMetaRole, self).__init__( - options=options, - cost_manager=cost_manager, - name=global_variables["name"], - profile=global_variables["profile"], - goal=global_variables["goal"], - constraints=global_variables["constraints"], - desc=global_variables["desc"], - **kwargs - ) - actions = [] - for m in opts.actions: - for k, v in m.items(): - v = Role.format_value(v, kwargs) - m[k] = v - for k, v in global_variables.items(): - if k not in m: - m[k] = v - - o = MetaActionOptions(**m) - o.set_default_template(opts.templates[o.template_ix]) - - act = MetaAction(options=options, action_options=o, llm=self._llm, **m) - actions.append(act) - self._init_actions(actions) - requirement_types = set() - for v in opts.requirement: - requirement_types.add(MetaAction.get_action_type(v)) - self._watch(requirement_types) - - async def _think(self) -> None: - """Everything will be done part by part.""" - if self._rc.todo is None: - self._set_state(0) - return True - - if self._rc.state + 1 < len(self._states): - self._set_state(self._rc.state + 1) - else: - self._rc.todo = None - return False - - async def _react(self) -> Message: - ret = Message(content="") - while True: - await self._think() - if self._rc.todo is None: - break - logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") - msg = await self._act() - if ret.content != '': - ret.content += "\n\n\n" - ret.content += msg.content - logger.info(ret.content) - await self.save(ret.content) - return ret - - async def save(self, content): - """Save teaching plan""" - output_filename = self.options.get("output_filename") - if not output_filename: - return - filename = ForkMetaRole.new_file_name(output_filename) - pathname = WORKSPACE_ROOT / "teaching_plan" - pathname.mkdir(exist_ok=True) - pathname = pathname / filename - try: - async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: - await writer.write(content) - except Exception as e: - logger.error(f'Save failed：{e}') - logger.info(f"Save to:{pathname}") - - @staticmethod - def new_file_name(lesson_title, ext=".md"): - """Create a related file name based on `lesson_title` and `ext`.""" - # Define the special characters that need to be replaced. - illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' - # Replace the special characters with underscores. - filename = re.sub(illegal_chars, '_', lesson_title) + ext - return re.sub(r'_+', '_', filename) \ No newline at end of file diff --git a/metagpt/roles/uml_meta_role_factory.py b/metagpt/roles/uml_meta_role_factory.py deleted file mode 100644 index 42071b0a6..000000000 --- a/metagpt/roles/uml_meta_role_factory.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : uml_meta_role_factory.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` -""" - -from metagpt.roles.fork_meta_role import ForkMetaRole -from metagpt.roles.uml_meta_role_options import UMLMetaRoleOptions - - -class UMLMetaRoleFactory: - """Factory of UML activity role classes""" - - @classmethod - def create_roles(cls, role_configs, **kwargs): - """Generate the flow of the project based on the configuration in the format of config/pattern/template.yaml. - - :param role_configs: `roles` field of template.yaml - :param kwargs: Parameters passed in format: `python your_script.py --param1=value1 --param2=value2` - - """ - roles = [] - for m in role_configs: - opt = UMLMetaRoleOptions(**m) - constructor = cls.CONSTRUCTORS.get(opt.role_type) - if constructor is None: - raise NotImplementedError( - f"{opt.role_type} is not implemented" - ) - r = constructor(role_options=m, **kwargs) - roles.append(r) - return roles - - CONSTRUCTORS = { - "fork": ForkMetaRole, - # TODO: add more activity node constructor here.. - } diff --git a/metagpt/roles/uml_meta_role_options.py b/metagpt/roles/uml_meta_role_options.py deleted file mode 100644 index 1d0fb322e..000000000 --- a/metagpt/roles/uml_meta_role_options.py +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : uml_meta_role_options.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` -""" - -from typing import List, Dict - -from pydantic import BaseModel - - -# `startup` field of config/pattern/template.yaml -class StartupConfig(BaseModel): - requirement: str - role: str - investment: float = 3.0 - n_round: int = 3 - - -# config/pattern/template.yaml -class ProjectConfig(BaseModel): - startup: StartupConfig - roles: List[Dict] - - -# element of `actions` field of config/pattern/template.yaml -class MetaActionOptions(BaseModel): - topic: str - name: str = "" - language: str = "Chinese" - template_ix: int = 0 - statements: List[str] = [] - template: str = "" - rsp_begin_tag: str = "" - rsp_end_tag: str = "" - - def set_default_template(self, v): - if not self.template: - self.template = v - - def format_prompt(self, **kwargs): - statements = "\n".join(self.statements) - opts = kwargs.copy() - opts["statements"] = statements - - from metagpt.roles import Role - prompt = Role.format_value(self.template, opts) - return prompt - - -# element of `roles` field of config/pattern/template.yaml -class UMLMetaRoleOptions(BaseModel): - role_type: str - name: str = "" - profile: str = "" - goal: str = "" - role: str = "" - constraints: str = "" - desc: str = "" - templates: List[str] = [] - output_filename: str = "" - actions: List - requirement: List diff --git a/tests/metagpt/actions/test_meta_action.py b/tests/metagpt/actions/test_meta_action.py deleted file mode 100644 index cbaf3456c..000000000 --- a/tests/metagpt/actions/test_meta_action.py +++ /dev/null @@ -1,51 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_meta_action.py -""" -from typing import Dict - -from pydantic import BaseModel - -from metagpt.actions.meta_action import MetaAction -from metagpt.roles.uml_meta_role_options import MetaActionOptions - - -def test_meta_action_create(): - class Inputs(BaseModel): - options: Dict - kwargs: Dict - expect_class_name: str - expect_prompt: str - - inputs = [ - { - "options": { - "topic": "TOPIC_A", - "name": "A", - "language": "XX", - "template_ix": 0, - "statements": ["Statement A", "Statement B"], - "template": "{statements}", - "rsp_begin_tag": "", - "rsp_end_tag": "" - }, - "kwargs": {}, - "expect_class_name": "TOPIC_A", - "expect_prompt": "\n".join(["Statement A", "Statement B"]), - } - ] - - for i in inputs: - seed = Inputs(**i) - opt = MetaActionOptions(**seed.options) - act = MetaAction(opt, **seed.kwargs) - assert seed.expect_prompt == act.prompt - t = MetaAction.get_action_type(seed.expect_class_name) - assert t.__name__ == seed.expect_class_name - - -if __name__ == '__main__': - test_meta_action_create() diff --git a/tests/metagpt/roles/test_fork_meta_role.py b/tests/metagpt/roles/test_fork_meta_role.py deleted file mode 100644 index 355197234..000000000 --- a/tests/metagpt/roles/test_fork_meta_role.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_fork_meta_role.py -""" -from typing import Dict - -from pydantic import BaseModel - -from metagpt.config import Config -from metagpt.provider.openai_api import CostManager -from metagpt.roles.fork_meta_role import ForkMetaRole - - -def test_creat_role(): - class Inputs(BaseModel): - role: Dict - action_count: int - - inputs = [ - { - "role": { - "role_type": "fork", - "name": "Lily", - "profile": "{teaching_language} Teacher", - "goal": "writing a {language} teaching plan part by part", - "constraints": "writing in {language}", - "role": "You are a {teaching_language} Teacher, named Lily, your goal is writing a {" - "teaching_language} teaching plan part by part, and the constraint is writing in {language}.", - "desc": "", - "output_filename": "teaching_plan_demo.md", - "requirement": ["TeachingPlanRequirement"], - "templates": [ - "Do not refer to the context of the previous conversation records, start the conversation " - "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" - "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " - "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " - "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " - "responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: " - "Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[" - "LESSON_BEGIN]\n{lesson}\n[LESSON_END]", - "Do not refer to the context of the previous conversation records, start the conversation " - "anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[" - "LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" " - "defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the " - "format requirements for your responses;\n\t\"Constraint\" defines the conditions that your " - "responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part " - "of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: " - "Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" " - "tags.\nAnswer options: Using proper markdown format from second-level header " - "format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END] " - ], - "actions": [ - { - "name": "", - "topic": "Title", - "language": "Chinese", - "statements": [ - "Statement: Find and return the title of the lesson only with \"# \" prefixed, without " - "anything else."], - "template_ix": 0}, - { - "name": "", - "topic": "Teaching Hours", - "language": "Chinese", - "statements": [], - "template_ix": 1, - "rsp_begin_tag": "[TEACHING_PLAN_BEGIN]", - "rsp_end_tag": "[TEACHING_PLAN_END]"} - ] - }, - "action_count": 2 - } - ] - - for i in inputs: - seed = Inputs(**i) - kwargs = { - "teaching_language": "AA", - "language": "BB" - } - runtime_options = Config().runtime_options - cost_manager = CostManager(options=runtime_options) - role = ForkMetaRole(runtime_options=runtime_options, cost_manager=cost_manager, role_options=seed.role, **kwargs) - assert role.action_count == 2 - assert "{" not in role.profile - assert "{" not in role.goal - assert "{" not in role.constraints - - -if __name__ == '__main__': - test_creat_role() diff --git a/tests/metagpt/roles/test_uml_meta_role_factory.py b/tests/metagpt/roles/test_uml_meta_role_factory.py deleted file mode 100644 index f59a30611..000000000 --- a/tests/metagpt/roles/test_uml_meta_role_factory.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_uml_meta_role_factory.py -""" -from typing import List, Dict - -from pydantic import BaseModel - -from metagpt.roles.uml_meta_role_factory import UMLMetaRoleFactory - - -def test_create_roles(): - class Inputs(BaseModel): - roles: List - kwargs: Dict - - inputs = [ - { - "roles": [ - { - "role_type": "fork", - "name": "Lily", - "profile": "{teaching_language} Teacher", - "goal": "writing a {language} teaching plan part by part", - "constraints": "writing in {language}", - "role": "You are a {teaching_language} Teacher, named Lily.", - "desc": "", - "output_filename": "teaching_plan_demo.md", - "requirement": ["TeachingPlanRequirement"], - "templates": ["Do 1 {statements}", "Do 2 {statements}"], - "actions": [ - { - "name": "", - "topic": "Title", - "language": "Chinese", - "statements": ["statement 1", "statement 2"]} - ], - "template_ix": 0 - } - ], - "kwargs": { - "teaching_language": "AA", - "language": "BB", - } - } - ] - - for i in inputs: - seed = Inputs(**i) - roles = UMLMetaRoleFactory.create_roles(seed.roles, **seed.kwargs) - assert len(roles) == 1 - assert "{" not in roles[0].profile - assert "{" not in roles[0].goal - assert roles[0].action_count == 1 - - -if __name__ == '__main__': - test_create_roles() diff --git a/tests/metagpt/roles/test_uml_meta_role_options.py b/tests/metagpt/roles/test_uml_meta_role_options.py deleted file mode 100644 index 1eb66c50e..000000000 --- a/tests/metagpt/roles/test_uml_meta_role_options.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/8 -@Author : mashenquan -@File : test_uml_meta_role_options.py -""" -from typing import List - -from pydantic import BaseModel - -from metagpt.roles.uml_meta_role_options import MetaActionOptions - - -def test_set_default_template(): - class Inputs(BaseModel): - statements: List - template: str - expect_prompt: str - - inputs = [ - { - "statements": ["Statement: 1", "Statement: 2"], - "template": "{statements}", - "expect_prompt": "Statement: 1\nStatement: 2" - } - ] - - for i in inputs: - seed = Inputs(**i) - opt = MetaActionOptions(topic="", statements=seed.statements) - assert opt.template == "" - opt.set_default_template(seed.template) - assert opt.template == seed.template - kwargs = {} - assert opt.format_prompt(**kwargs) == seed.expect_prompt - - -if __name__ == '__main__': - test_set_default_template() From f33af9dbc9d7af71aafacb6aa51b936eaf3e56c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 22:24:25 +0800 Subject: [PATCH 094/378] fixbug: skill_yaml_file_name --- metagpt/learn/skill_loader.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 46ead728d..71535f310 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -32,9 +32,10 @@ class SkillsDeclaration(BaseModel): class SkillLoader: - def __init__(self): - skill_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" - with open(str(skill_file_name), 'r') as file: + def __init__(self, skill_yaml_file_name: Path = None): + if not skill_yaml_file_name: + skill_yaml_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" + with open(str(skill_yaml_file_name), 'r') as file: skills = yaml.safe_load(file) self._skills = SkillsDeclaration(**skills) From 1545a702ccd4610c14acd818ae8fc6a19fd8d84d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 26 Aug 2023 22:28:41 +0800 Subject: [PATCH 095/378] fixbug: skill_yaml_file_name --- metagpt/actions/meta_action.py | 64 ---------------------------------- metagpt/roles/assistant.py | 5 ++- 2 files changed, 4 insertions(+), 65 deletions(-) delete mode 100644 metagpt/actions/meta_action.py diff --git a/metagpt/actions/meta_action.py b/metagpt/actions/meta_action.py deleted file mode 100644 index 4c52e7cfd..000000000 --- a/metagpt/actions/meta_action.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/7 -@Author : mashenquan -@File : meta_action.py -@Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the - ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to - make these symbols configurable and standardized, making the process of building flows more convenient. - For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` - This file defines a meta action capable of generating arbitrary actions at runtime based on a - configuration file. -""" - -from typing import Type - -from metagpt.actions import Action -from metagpt.logs import logger -from metagpt.roles.uml_meta_role_options import MetaActionOptions -from metagpt.schema import Message - - -class MetaAction(Action): - def __init__(self, options, action_options: MetaActionOptions, llm=None, **kwargs): - super(MetaAction, self).__init__(options=options, - name=action_options.name, - context=kwargs.get("context"), - llm=llm) - self.prompt = action_options.format_prompt(**kwargs) - self.action_options = action_options - self.kwargs = kwargs - - def __str__(self): - """Return `topic` value when str()""" - return self.action_options.topic - - def __repr__(self): - """Show `topic` value when debug""" - return self.action_options.topic - - async def run(self, messages, *args, **kwargs): - if len(messages) < 1 or not isinstance(messages[0], Message): - raise ValueError("Invalid args, a tuple of List[Message] is expected") - - logger.debug(self.prompt) - rsp = await self._aask(prompt=self.prompt) - logger.debug(rsp) - self._set_result(rsp) - return self.rsp - - def _set_result(self, rsp): - if self.action_options.rsp_begin_tag and self.action_options.rsp_begin_tag in rsp: - ix = rsp.index(self.action_options.rsp_begin_tag) - rsp = rsp[ix + len(self.action_options.rsp_begin_tag):] - if self.action_options.rsp_end_tag and self.action_options.rsp_end_tag in rsp: - ix = rsp.index(self.action_options.rsp_end_tag) - rsp = rsp[0:ix] - self.rsp = rsp.strip() - - @staticmethod - def get_action_type(topic: str): - """Create a runtime :class:`Action` subclass""" - action_type: Type["Action"] = type(topic, (Action,), {"name": topic}) - return action_type diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c001d69f0..a3af715e3 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -14,6 +14,7 @@ """ import asyncio +from pathlib import Path from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction @@ -29,6 +30,7 @@ from metagpt.schema import Message DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 BRAIN_MEMORY = "BRAIN_MEMORY" +SKILL_PATH = "SKILL_PATH" class Assistant(Role): @@ -40,7 +42,8 @@ class Assistant(Role): goal=goal, constraints=constraints, desc=desc, *args, **kwargs) brain_memory = options.get(BRAIN_MEMORY) self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() - self.skills = SkillLoader() + skill_path = Path(options.get(SKILL_PATH)) if options.get(SKILL_PATH) else None + self.skills = SkillLoader(skill_yaml_file_name=skill_path) async def think(self) -> bool: """Everything will be done part by part.""" From ee77d4b0fb2ce865b59de2a6095d01c9ab695f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 10:41:34 +0800 Subject: [PATCH 096/378] feat: +exported function --- metagpt/roles/role.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 47f494c69..286c87eb1 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -328,9 +328,16 @@ class Role: self._rc.todo = act async def think(self) -> bool: - return await self._think() + """The exported `think` function""" + has_action = await self._think() + if not has_action: + return False + if not self._rc.todo: + return False + return True async def act(self) -> ActionOutput: + """The exported `act` function""" msg = await self._act() return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) From 93d6bc6569e4e011d7823a21bea018d7b05ef57b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 11:13:32 +0800 Subject: [PATCH 097/378] feat: +todo_description --- metagpt/roles/role.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 286c87eb1..c57bf4f43 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -341,3 +341,11 @@ class Role: msg = await self._act() return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) + + @property + def todo_description(self): + if not self._rc or not self._rc.todo: + return "" + if self._rc.todo.desc: + return self._rc.todo.desc + return f"{self._rc.todo.__class__}" From 6d3f2acddbcb7a68004dbcb7a228d19918ce22ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 11:19:59 +0800 Subject: [PATCH 098/378] feat: +todo_description --- metagpt/roles/role.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index c57bf4f43..ed02575db 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -348,4 +348,4 @@ class Role: return "" if self._rc.todo.desc: return self._rc.todo.desc - return f"{self._rc.todo.__class__}" + return f"{type(self._rc.todo).__name__}" From 5a03ff20ce65e353955a5209ac65e91edd007fdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 11:58:32 +0800 Subject: [PATCH 099/378] fixbug: call skill in api --- metagpt/roles/assistant.py | 5 +++-- metagpt/utils/common.py | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a3af715e3..3924039b5 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -98,14 +98,15 @@ class Assistant(Role): async def skill_handler(self, text, **kwargs) -> bool: last_talk = kwargs.get("last_talk") skill = self.skills.get_skill(text) - logger.info(f"skill not found: {text}") if not skill: + logger.info(f"skill not found: {text}") return await self.talk_handler(text=last_talk, **kwargs) action = ArgumentsParingAction(options=self.options, skill=skill, llm=self._llm, **kwargs) await action.run(**kwargs) if action.args is None: return await self.talk_handler(text=last_talk, **kwargs) - action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm) + action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm, name=skill.name, + desc=skill.description) self.add_to_do(action) return True diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index ea6af7e7c..a6e4dc20d 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -260,9 +260,16 @@ def parse_recipient(text): return recipient.group(1) if recipient else "" -def initialize_environment(): +def initialize_environment(options=None): """Load `config/config.yaml` to `os.environ`""" + if options: + for k, v in options.items(): + os.environ[k] = str(v) + return + yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" + if not yaml_file_path.exists(): + return with open(str(yaml_file_path), "r") as yaml_file: data = yaml.safe_load(yaml_file) for k, v in data.items(): From 4fddfbab581b28736ef851f99bee14f5d1385179 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 13:22:34 +0800 Subject: [PATCH 100/378] fixbug: No user feedback, unsure if past conversation is finished. --- metagpt/memory/brain_memory.py | 2 +- metagpt/roles/assistant.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 422c096f3..9d1b038bb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -53,6 +53,6 @@ class BrainMemory(pydantic.BaseModel): @property def last_talk(self): if len(self.history) == 0 or not self.history[-1].is_contain_tags([MessageType.Talk.value]): - return "" + return None return self.history[-1].content diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 3924039b5..1e503857a 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -113,6 +113,8 @@ class Assistant(Role): async def refine_memory(self) -> str: history_text = self.memory.history_text last_talk = self.memory.last_talk + if last_talk is None: # No user feedback, unsure if past conversation is finished. + return None if history_text == "": return last_talk history_summary = await self._llm.get_context_title(history_text, max_words=20) From 903e89cec36b7d07e1bd52b9894e7c6b7131ca6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 13:27:13 +0800 Subject: [PATCH 101/378] fixbug: No user feedback, unsure if past conversation is finished. --- metagpt/roles/assistant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 1e503857a..199cdcafd 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -130,8 +130,8 @@ class Assistant(Role): from metagpt.provider.openai_api import OpenAIGPTAPI return OpenAIGPTAPI.extract_info(input_string) - def get_memory(self) -> str: - return self.memory.json() + def get_memory(self, exclude=None) -> str: + return self.memory.json(exclude=exclude) def load_memory(self, jsn): try: From 3e9151e52e331978548ba6a9a6527e5569991a65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 27 Aug 2023 15:11:28 +0800 Subject: [PATCH 102/378] fixbug: brain memory serialize --- metagpt/memory/brain_memory.py | 30 ++++++------ metagpt/roles/assistant.py | 2 +- metagpt/schema.py | 17 ++++++- tests/metagpt/memory/test_brain_memory.py | 57 +++++++++++++++++++++++ 4 files changed, 90 insertions(+), 16 deletions(-) create mode 100644 tests/metagpt/memory/test_brain_memory.py diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 9d1b038bb..cb67fea8e 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -1,10 +1,11 @@ from enum import Enum -from typing import List +from typing import List, Dict import pydantic from metagpt import Message + class MessageType(Enum): Talk = "TALK" Solution = "SOLUTION" @@ -14,29 +15,28 @@ class MessageType(Enum): class BrainMemory(pydantic.BaseModel): - history: List[Message] = [] - stack: List[Message] = [] - solution: List[Message] = [] - knowledge: List[Message] = [] - + history: List[Dict] = [] + stack: List[Dict] = [] + solution: List[Dict] = [] + knowledge: List[Dict] = [] def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) - self.history.append(msg) + self.history.append(msg.dict()) def add_answer(self, msg: Message): msg.add_tag(MessageType.Answer.value) - self.history.append(msg) + self.history.append(msg.dict()) def get_knowledge(self) -> str: - texts = [k.content for k in self.knowledge] + texts = [Message(**m).content for m in self.knowledge] return "\n".join(texts) @property def history_text(self): if len(self.history) == 0: return "" - texts = [m.content for m in self.history[:-1]] + texts = [Message(**m).content for m in self.history[:-1]] return "\n".join(texts) def move_to_solution(self): @@ -44,7 +44,7 @@ class BrainMemory(pydantic.BaseModel): return msgs = self.history[:-1] self.solution.extend(msgs) - if not self.history[-1].is_contain(MessageType.Talk.value): + if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): self.solution.append(self.history[-1]) self.history = [] else: @@ -52,7 +52,9 @@ class BrainMemory(pydantic.BaseModel): @property def last_talk(self): - if len(self.history) == 0 or not self.history[-1].is_contain_tags([MessageType.Talk.value]): + if len(self.history) == 0: return None - return self.history[-1].content - + last_msg = Message(**self.history[-1]) + if not last_msg.is_contain(MessageType.Talk.value): + return None + return last_msg.content diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 199cdcafd..4519fcdb8 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -77,7 +77,7 @@ class Assistant(Role): return output async def talk(self, text): - self.memory.add_talk(Message(content=text, tags=set([MessageType.Talk.value]))) + self.memory.add_talk(Message(content=text)) async def _plan(self, rsp: str, **kwargs) -> bool: skill, text = Assistant.extract_info(input_string=rsp) diff --git a/metagpt/schema.py b/metagpt/schema.py index 909313886..4d7f0cc21 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -70,6 +70,22 @@ class Message: def is_contain(self, tag): return self.is_contain_tags([tag]) + def dict(self): + """pydantic-like `dict` function""" + full = { + "instruct_content": self.instruct_content, + "cause_by": self.cause_by, + "sent_from": self.sent_from, + "send_to": self.send_to, + "tags": self.tags + } + + m = {"content": self.content} + for k, v in full.items(): + if v: + m[k] = v + return m + @dataclass class UserMessage(Message): @@ -101,7 +117,6 @@ class AIMessage(Message): super().__init__(content, 'assistant') - if __name__ == '__main__': test_content = 'test_message' msgs = [ diff --git a/tests/metagpt/memory/test_brain_memory.py b/tests/metagpt/memory/test_brain_memory.py new file mode 100644 index 000000000..b5fc942ca --- /dev/null +++ b/tests/metagpt/memory/test_brain_memory.py @@ -0,0 +1,57 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/27 +@Author : mashenquan +@File : test_brain_memory.py +""" +import json +from typing import List + +import pydantic + +from metagpt.memory.brain_memory import BrainMemory +from metagpt.schema import Message + + +def test_json(): + class Input(pydantic.BaseModel): + history: List[str] + solution: List[str] + knowledge: List[str] + stack: List[str] + + inputs = [ + { + "history": ["a", "b"], + "solution": ["c"], + "knowledge": ["d", "e"], + "stack": ["f"] + } + ] + + for i in inputs: + v = Input(**i) + bm = BrainMemory() + for h in v.history: + msg = Message(content=h) + bm.history.append(msg.dict()) + for h in v.solution: + msg = Message(content=h) + bm.solution.append(msg.dict()) + for h in v.knowledge: + msg = Message(content=h) + bm.knowledge.append(msg.dict()) + for h in v.stack: + msg = Message(content=h) + bm.stack.append(msg.dict()) + s = bm.json() + m = json.loads(s) + bm = BrainMemory(**m) + assert bm + for v in bm.history: + msg = Message(**v) + assert msg + +if __name__ == '__main__': + test_json() \ No newline at end of file From 9b890275c4c4a2e1580556113b0928ca871da838 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 10:36:13 +0800 Subject: [PATCH 103/378] feat: +x-prerequisite --- .well-known/metagpt_oas3_api.yaml | 14 ++++++++++++++ metagpt/tools/metagpt_text_to_image.py | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index a226181a5..56c6f42d5 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -12,6 +12,11 @@ servers: paths: /tts/azsure: + x-prerequisite: + - name: AZURE_TTS_SUBSCRIPTION_KEY + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + - name: AZURE_TTS_REGION + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" post: summary: "Convert Text to Base64-encoded .wav File Stream" description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" @@ -69,6 +74,9 @@ paths: description: "Internal Server Error" /txt2img/openai: + x-prerequisite: + - name: OPENAI_API_KEY + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: "Convert Text to Base64-encoded Image Data Stream" operationId: openai_text_to_image.oas3_openai_text_to_image @@ -107,6 +115,9 @@ paths: '500': description: "Internal Server Error" /txt2embedding/openai: + x-prerequisite: + - name: OPENAI_API_KEY + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: Text to embedding operationId: openai_text_to_embedding.oas3_openai_text_to_embedding @@ -146,6 +157,9 @@ paths: $ref: "#/components/schemas/Error" /txt2image/metagpt: + x-prerequisite: + - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + description: "Model url." post: summary: "Text to Image" description: "Generate an image from the provided text using the MetaGPT Text-to-Image API." diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 674ff283a..8588462d3 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -98,7 +98,7 @@ def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): if not text: return "" if not model_url: - model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL_URL') return MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) From 13eddeae2fab883106be8547b1b84f8c42903775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 10:42:54 +0800 Subject: [PATCH 104/378] feat: +x-prerequisite --- .well-known/skills.yaml | 16 ++++++++++------ metagpt/learn/text_to_image.py | 2 +- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 7a035910c..06b9ffd0c 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -4,9 +4,11 @@ entities: - name: text_to_speech description: Text-to-speech id: text_to_speech.text_to_speech - requisite: - - AZURE_TTS_SUBSCRIPTION_KEY - - AZURE_TTS_REGION + x-prerequisite: + - name: AZURE_TTS_SUBSCRIPTION_KEY + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + - name: AZURE_TTS_REGION + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" arguments: text: 'The text used for voice conversion. Required.' lang: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States). The optional parameter are "English", "Chinese". Default value: "Chinese".' @@ -27,9 +29,11 @@ entities: - name: text_to_image description: Create a drawing based on the text. id: text_to_image.text_to_image - requisite: - - OPENAI_API_KEY - - METAGPT_TEXT_TO_IMAGE_MODEL + x-prerequisite: + - name: OPENAI_API_KEY + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + description: "Model url." arguments: text: 'The text used for image conversion. Required.' size_type: 'Default value: "512x512".' diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 2f946e239..d245b06db 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -28,7 +28,7 @@ def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url """ initialize_environment() image_declaration = "data:image/png;base64," - if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL") or model_url: + if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL_URL") or model_url: data = oas3_metagpt_text_to_image(text, size_type, model_url) return image_declaration + data if data else "" if os.environ.get("OPENAI_API_KEY") or openai_api_key: From aaf18d2641113bb410a91776948b7fcf810ef2ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 11:04:28 +0800 Subject: [PATCH 105/378] feat: +x-prerequisite --- .well-known/ai-plugin.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.well-known/ai-plugin.json b/.well-known/ai-plugin.json index 44e8435f2..ac0178fd0 100644 --- a/.well-known/ai-plugin.json +++ b/.well-known/ai-plugin.json @@ -9,10 +9,10 @@ }, "api": { "type": "openapi", - "url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/.well-known/metagpt_oas3_api.yaml", + "url": "https://github.com/iorisa/MetaGPT/blob/feature/assistant_role/.well-known/metagpt_oas3_api.yaml", "has_user_authentication": false }, - "logo_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/resources/MetaGPT-logo.png", + "logo_url": "https://github.com/geekan/MetaGPT/blob/main/docs/resources/MetaGPT-logo.png", "contact_email": "mashenquan@fuzhi.cn", - "legal_info_url": "https://github.com/iorisa/MetaGPT/blob/feature/oas3/docs/README_CN.md" + "legal_info_url": "https://github.com/geekan/MetaGPT/blob/main/docs/README_CN.md" } \ No newline at end of file From c67789756147d84d785f09b0e3a33497442d91cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 11:48:38 +0800 Subject: [PATCH 106/378] fixbug: runtime options --- metagpt/roles/role.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index ed02575db..f605f5010 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -14,7 +14,9 @@ from __future__ import annotations from typing import Iterable, Type, Dict from pydantic import BaseModel, Field -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + +from metagpt.config import Config +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager from metagpt.actions import Action, ActionOutput from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory @@ -100,7 +102,11 @@ class RoleContext(BaseModel): class Role: """Role/Proxy""" - def __init__(self, options, cost_manager, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + def __init__(self, options=None, cost_manager=None, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + if not options: + options = Config().runtime_options + if not cost_manager: + cost_manager = CostManager(*options) self._options = Role.supply_options(options=kwargs, default_options=options) name = Role.format_value(name, self._options) From ac744062609d6218b5cac72be5916067667f9d34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 11:55:09 +0800 Subject: [PATCH 107/378] =?UTF-8?q?fixbug:=20+=E7=BC=BA=E7=9C=81=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/action.py | 5 +++-- metagpt/roles/role.py | 7 +++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 86a6664ba..10579d4f4 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -12,13 +12,14 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput +from metagpt.config import Config from metagpt.utils.common import OutputParser from metagpt.logs import logger class Action(ABC): - def __init__(self, options, name: str = '', context=None, llm=None): - self.options = options + def __init__(self, options=None, name: str = '', context=None, llm=None): + self.options = options or Config().runtime_options self.name: str = name self.llm = llm self.context = context diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index f605f5010..4f46bb973 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -103,10 +103,9 @@ class Role: """Role/Proxy""" def __init__(self, options=None, cost_manager=None, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): - if not options: - options = Config().runtime_options - if not cost_manager: - cost_manager = CostManager(*options) + options = options or Config().runtime_options + cost_manager = cost_manager or CostManager(*options) + self._options = Role.supply_options(options=kwargs, default_options=options) name = Role.format_value(name, self._options) From b410b9352078cbcf35df8690f241c38311df4840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 12:01:34 +0800 Subject: [PATCH 108/378] =?UTF-8?q?fixbug:=20+=E7=BC=BA=E7=9C=81=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 4519fcdb8..dae516795 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -3,7 +3,7 @@ """ @Time : 2023/8/7 @Author : mashenquan -@File : fork_meta_role.py +@File : assistant.py @Desc : I am attempting to incorporate certain symbol concepts from UML into MetaGPT, enabling it to have the ability to freely construct flows through symbol concatenation. Simultaneously, I am also striving to make these symbols configurable and standardized, making the process of building flows more convenient. From f17660b12251ddb362d4f3233589093cb61c8cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 12:16:01 +0800 Subject: [PATCH 109/378] fixbug: get_memory --- metagpt/roles/assistant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index dae516795..d6f52e4e4 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -130,8 +130,8 @@ class Assistant(Role): from metagpt.provider.openai_api import OpenAIGPTAPI return OpenAIGPTAPI.extract_info(input_string) - def get_memory(self, exclude=None) -> str: - return self.memory.json(exclude=exclude) + def get_memory(self) -> str: + return self.memory.json() def load_memory(self, jsn): try: From 6acf3f628238f960d8771efcd6ad9f035c3af7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 12:36:31 +0800 Subject: [PATCH 110/378] fixbug: get_memory --- metagpt/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index 4d7f0cc21..ce08455fc 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -74,7 +74,6 @@ class Message: """pydantic-like `dict` function""" full = { "instruct_content": self.instruct_content, - "cause_by": self.cause_by, "sent_from": self.sent_from, "send_to": self.send_to, "tags": self.tags From 6794645ff63e6b28f17492907c60755c447d2c24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 15:42:47 +0800 Subject: [PATCH 111/378] =?UTF-8?q?feat:=20=E6=94=B9=E5=BC=82=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/learn/text_to_embedding.py | 4 ++-- metagpt/learn/text_to_image.py | 7 +++--- metagpt/learn/text_to_speech.py | 4 ++-- metagpt/tools/azure_tts.py | 17 +++++++------ metagpt/tools/hello.py | 2 +- metagpt/tools/metagpt_oas3_api_svc.py | 2 +- metagpt/tools/metagpt_text_to_image.py | 13 +++++----- metagpt/tools/openai_text_to_embedding.py | 19 ++++++++------- metagpt/tools/openai_text_to_image.py | 24 ++++++++++--------- requirements.txt | 1 + tests/metagpt/learn/test_text_to_embedding.py | 4 ++-- tests/metagpt/learn/test_text_to_image.py | 11 +++++++-- tests/metagpt/learn/test_text_to_speech.py | 11 +++++++-- tests/metagpt/tools/test_azure_tts.py | 9 ++++--- 14 files changed, 78 insertions(+), 50 deletions(-) diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 6d0cefcdb..5c08ef0b9 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -16,7 +16,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to Embedding", description="Convert the text into embeddings.", requisite="`OPENAI_API_KEY`") -def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): +async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): """Text to embedding :param text: The text used for embedding. @@ -26,5 +26,5 @@ def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", * """ initialize_environment() if os.environ.get("OPENAI_API_KEY") or openai_api_key: - return oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) + return await oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) raise EnvironmentError diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index d245b06db..db9844c71 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -17,7 +17,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to image", description="Create a drawing based on the text.", requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") -def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): +async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): """Text to image :param text: The text used for image conversion. @@ -29,10 +29,11 @@ def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url initialize_environment() image_declaration = "data:image/png;base64," if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL_URL") or model_url: - data = oas3_metagpt_text_to_image(text, size_type, model_url) + data = await oas3_metagpt_text_to_image(text, size_type, model_url) return image_declaration + data if data else "" + if os.environ.get("OPENAI_API_KEY") or openai_api_key: - data = oas3_openai_text_to_image(text, size_type, openai_api_key) + data = await oas3_openai_text_to_image(text, size_type, openai_api_key) return image_declaration + data if data else "" raise EnvironmentError diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 90dd878a1..e5eb3d488 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -16,7 +16,7 @@ from metagpt.utils.common import initialize_environment @skill_metadata(name="Text to speech", description="Text-to-speech", requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") -def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", +async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region="", **kwargs): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -35,7 +35,7 @@ def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affect audio_declaration = "data:audio/wav;base64," if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ (subscription_key and region): - data = oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) return audio_declaration + data if data else data raise EnvironmentError diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 21e8f1b6c..1fd36e78c 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -6,6 +6,7 @@ @File : azure_tts.py @Desc : azure TTS OAS3 api, which provides text-to-speech functionality """ +import asyncio from pathlib import Path from uuid import uuid4 import base64 @@ -14,7 +15,7 @@ import sys sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.utils.common import initialize_environment from metagpt.logs import logger - +from aiofile import async_open from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer import os @@ -31,7 +32,7 @@ class AzureTTS: self.region = region if region else os.environ.get('AZURE_TTS_REGION') # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles - def synthesize_speech(self, lang, voice, text, output_file): + async def synthesize_speech(self, lang, voice, text, output_file): speech_config = SpeechConfig( subscription=self.subscription_key, region=self.region) speech_config.speech_synthesis_voice_name = voice @@ -61,7 +62,7 @@ class AzureTTS: # Export -def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): +async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key="", region=""): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -95,9 +96,9 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key tts = AzureTTS(subscription_key=subscription_key, region=region) filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav") try: - tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) - with open(str(filename), mode="rb") as reader: - data = reader.read() + await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) + async with async_open(filename, mode="rb") as reader: + data = await reader.read() base64_string = base64.b64encode(data).decode('utf-8') filename.unlink() except Exception as e: @@ -110,5 +111,7 @@ def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscription_key if __name__ == "__main__": initialize_environment() - v = oas3_azsure_tts("测试，test") + loop = asyncio.new_event_loop() + v = loop.create_task(oas3_azsure_tts("测试，test")) + loop.run_until_complete(v) print(v) diff --git a/metagpt/tools/hello.py b/metagpt/tools/hello.py index e1bad6456..2eb4c31f0 100644 --- a/metagpt/tools/hello.py +++ b/metagpt/tools/hello.py @@ -17,7 +17,7 @@ import connexion # openapi implement -def post_greeting(name: str) -> str: +async def post_greeting(name: str) -> str: return f"Hello {name}\n" diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 277d41dfb..624bb7d93 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -20,7 +20,7 @@ def oas_http_svc(): """Start the OAS 3.0 OpenAPI HTTP service""" initialize_environment() - app = connexion.FlaskApp(__name__, specification_dir='../../.well-known/') + app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") app.run(port=8080) diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 8588462d3..bc551134a 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -12,6 +12,7 @@ import sys from pathlib import Path from typing import List, Dict +import aiohttp import requests from pydantic import BaseModel @@ -27,7 +28,7 @@ class MetaGPTText2Image: """ self.model_url = model_url if model_url else os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') - def text_2_image(self, text, size_type="512x512"): + async def text_2_image(self, text, size_type="512x512"): """Text to image :param text: The text used for image conversion. @@ -75,9 +76,9 @@ class MetaGPTText2Image: parameters: Dict try: - response = requests.post(self.model_url, headers=headers, json=data) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - result = ImageResult(**response.json()) + async with aiohttp.ClientSession() as session: + async with session.post(self.model_url, headers=headers, json=data) as response: + result = ImageResult(**await response.json()) if len(result.images) == 0: return "" return result.images[0] @@ -87,7 +88,7 @@ class MetaGPTText2Image: # Export -def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): +async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): """Text to image :param text: The text used for image conversion. @@ -99,7 +100,7 @@ def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url=""): return "" if not model_url: model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL_URL') - return MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) + return await MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) if __name__ == "__main__": diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 9eddd5bc1..119eb35b6 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -7,10 +7,12 @@ @Desc : OpenAI Text-to-Embedding OAS3 api, which provides text-to-embedding functionality. For more details, checkout: `https://platform.openai.com/docs/api-reference/embeddings/object` """ +import asyncio import os from pathlib import Path from typing import List +import aiohttp import requests from pydantic import BaseModel import sys @@ -47,7 +49,7 @@ class OpenAIText2Embedding: """ self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') - def text_2_embedding(self, text, model="text-embedding-ada-002"): + async def text_2_embedding(self, text, model="text-embedding-ada-002"): """Text to embedding :param text: The text used for embedding. @@ -61,16 +63,16 @@ class OpenAIText2Embedding: } data = {"input": text, "model": model} try: - response = requests.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - return response.json() + async with aiohttp.ClientSession() as session: + async with session.post("https://api.openai.com/v1/embeddings", headers=headers, json=data) as response: + return await response.json() except requests.exceptions.RequestException as e: logger.error(f"An error occurred:{e}") return {} # Export -def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): +async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_api_key=""): """Text to embedding :param text: The text used for embedding. @@ -82,11 +84,12 @@ def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", openai_a return "" if not openai_api_key: openai_api_key = os.environ.get("OPENAI_API_KEY") - return OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) + return await OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) if __name__ == "__main__": initialize_environment() - - v = oas3_openai_text_to_embedding("Panda emoji") + loop = asyncio.new_event_loop() + v = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) + loop.run_until_complete(v) print(v) diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 6ec96d166..cd48c62af 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -12,6 +12,7 @@ import sys from pathlib import Path from typing import List +import aiohttp import requests from pydantic import BaseModel @@ -27,7 +28,7 @@ class OpenAIText2Image: """ self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') - def text_2_image(self, text, size_type="1024x1024"): + async def text_2_image(self, text, size_type="1024x1024"): """Text to image :param text: The text used for image conversion. @@ -48,27 +49,28 @@ class OpenAIText2Image: } data = {"prompt": text, "n": 1, "size": size_type} try: - response = requests.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - result = ImageResult(**response.json()) + async with aiohttp.ClientSession() as session: + async with session.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) as response: + result = ImageResult(** await response.json()) except requests.exceptions.RequestException as e: logger.error(f"An error occurred:{e}") return "" if len(result.data) > 0: - return OpenAIText2Image.get_image_data(result.data[0].url) + return await OpenAIText2Image.get_image_data(result.data[0].url) return "" @staticmethod - def get_image_data(url): + async def get_image_data(url): """Fetch image data from a URL and encode it as Base64 :param url: Image url :return: Base64-encoded image data. """ try: - response = requests.get(url) - response.raise_for_status() # Raise an exception for 4xx or 5xx responses - image_data = response.content + async with aiohttp.ClientSession() as session: + async with session.get(url) as response: + response.raise_for_status() # 如果是 4xx 或 5xx 响应，会引发异常 + image_data = await response.read() base64_image = base64.b64encode(image_data).decode("utf-8") return base64_image @@ -78,7 +80,7 @@ class OpenAIText2Image: # Export -def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): +async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key=""): """Text to image :param text: The text used for image conversion. @@ -90,7 +92,7 @@ def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_api_key return "" if not openai_api_key: openai_api_key = os.environ.get("OPENAI_API_KEY") - return OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) + return await OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 70f2a3809..ed3f755c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,3 +41,4 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.30.0 +aiofile \ No newline at end of file diff --git a/tests/metagpt/learn/test_text_to_embedding.py b/tests/metagpt/learn/test_text_to_embedding.py index c85e5dde8..d81a8ac1c 100644 --- a/tests/metagpt/learn/test_text_to_embedding.py +++ b/tests/metagpt/learn/test_text_to_embedding.py @@ -8,11 +8,11 @@ """ import asyncio -import base64 from pydantic import BaseModel from metagpt.learn.text_to_embedding import text_to_embedding +from metagpt.tools.openai_text_to_embedding import ResultEmbedding async def mock_text_to_embedding(): @@ -25,7 +25,7 @@ async def mock_text_to_embedding(): for i in inputs: seed = Input(**i) - data = text_to_embedding(seed.input) + data = await text_to_embedding(seed.input) v = ResultEmbedding(**data) assert len(v.data) > 0 diff --git a/tests/metagpt/learn/test_text_to_image.py b/tests/metagpt/learn/test_text_to_image.py index 545c8a3ef..c359797de 100644 --- a/tests/metagpt/learn/test_text_to_image.py +++ b/tests/metagpt/learn/test_text_to_image.py @@ -25,10 +25,17 @@ async def mock_text_to_image(): for i in inputs: seed = Input(**i) - base64_data = text_to_image(seed.input) + base64_data = await text_to_image(seed.input) assert base64_data != "" print(f"{seed.input} -> {base64_data}") - assert base64.b64decode(base64_data, validate=True) + flags = ";base64," + assert flags in base64_data + ix = base64_data.find(flags) + len(flags) + declaration = base64_data[0: ix] + assert declaration + data = base64_data[ix:] + assert data + assert base64.b64decode(data, validate=True) def test_suite(): diff --git a/tests/metagpt/learn/test_text_to_speech.py b/tests/metagpt/learn/test_text_to_speech.py index dbb599e38..68de5a3b2 100644 --- a/tests/metagpt/learn/test_text_to_speech.py +++ b/tests/metagpt/learn/test_text_to_speech.py @@ -24,10 +24,17 @@ async def mock_text_to_speech(): for i in inputs: seed = Input(**i) - base64_data = text_to_speech(seed.input) + base64_data = await text_to_speech(seed.input) assert base64_data != "" print(f"{seed.input} -> {base64_data}") - assert base64.b64decode(base64_data, validate=True) + flags = ";base64," + assert flags in base64_data + ix = base64_data.find(flags) + len(flags) + declaration = base64_data[0: ix] + assert declaration + data = base64_data[ix:] + assert data + assert base64.b64decode(data, validate=True) def test_suite(): diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 49dd7eed1..41d429109 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -7,6 +7,7 @@ @Modified By: mashenquan, 2023-8-9, add more text formatting options @Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ +import asyncio import sys from pathlib import Path @@ -19,7 +20,7 @@ from metagpt.utils.common import initialize_environment def test_azure_tts(): initialize_environment() - azure_tts = AzureTTS() + azure_tts = AzureTTS(subscription_key="", region="") text = """ 女儿看见父亲走了进来，问道： @@ -33,11 +34,13 @@ def test_azure_tts(): path = WORKSPACE_ROOT / "tts" path.mkdir(exist_ok=True, parents=True) filename = path / "girl.wav" - result = azure_tts.synthesize_speech( + loop = asyncio.new_event_loop() + v = loop.create_task(azure_tts.synthesize_speech( lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, - output_file=str(filename)) + output_file=str(filename))) + result = loop.run_until_complete(v) print(result) From 3a1ebf19b7858f3d3156a7d29767200b23db5199 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 16:48:59 +0800 Subject: [PATCH 112/378] feat: +OPTIONS --- metagpt/config.py | 73 ++++++++++++++++++++++------------------------- metagpt/const.py | 3 ++ 2 files changed, 37 insertions(+), 39 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index 31488b466..ceaa582e2 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -1,18 +1,17 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- """ -@Desc: Provide configuration, singleton. -@Modified By: mashenquan, replace `CONFIG` with `os.environ` to support personal config - `os.environ` doesn't support personalization, while `Config` does. - Hence, the parameter reading priority is `Config` first, and if not found, then `os.environ`. -@Modified By: mashenquan, 2023/8/23. Add `options` to `Config.__init__` to support externally specified options. +Provide configuration, singleton. +@Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ import os +from copy import deepcopy +from typing import Any import openai import yaml -from metagpt.const import PROJECT_ROOT +from metagpt.const import PROJECT_ROOT, OPTIONS from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.singleton import Singleton @@ -30,34 +29,26 @@ class NotConfiguredException(Exception): super().__init__(self.message) -class Config: +class Config(metaclass=Singleton): """ - For example: - - ```python + Usual Usage: config = Config("config.yaml") secret_key = config.get_key("MY_SECRET_KEY") print("Secret key:", secret_key) - ``` """ + _instance = None key_yaml_file = PROJECT_ROOT / "config/key.yaml" default_yaml_file = PROJECT_ROOT / "config/config.yaml" - def __init__(self, yaml_file=default_yaml_file, options=None): - self._configs = {} - self._init_with_config_files_and_env(self._configs, yaml_file) - if options: - self._configs.update(options) - self._parse() - - def _parse(self): + def __init__(self, yaml_file=default_yaml_file): + self._init_with_config_files_and_env(yaml_file) logger.info("Config loading done.") self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -94,41 +85,45 @@ class Config: self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") - def _init_with_config_files_and_env(self, configs: dict, yaml_file): - """Load in decreasing priority from `config/key.yaml`, `config/config.yaml`, and environment variables.""" - configs.update(os.environ) + def _init_with_config_files_and_env(self, yaml_file): + """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" + configs = dict(os.environ) for _yaml_file in [yaml_file, self.key_yaml_file]: if not _yaml_file.exists(): continue - # Load local YAML file. + # 加载本地 YAML 文件 with open(_yaml_file, "r", encoding="utf-8") as file: yaml_data = yaml.safe_load(file) if not yaml_data: continue configs.update(yaml_data) + OPTIONS.set(configs) - def _get(self, *args, **kwargs): - return self._configs.get(*args, **kwargs) + @staticmethod + def _get(*args, **kwargs): + m = OPTIONS.get() + return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Retrieve value from `config/key.yaml`, `config/config.yaml`, and environment variables. - Raise an error if not found.""" + """Retrieve values from config/key.yaml, config/config.yaml, and environment variables. Throw an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") return value - @property - def runtime_options(self): - """Runtime key-value configuration parameters.""" - opts = {} - for k, v in self._configs.items(): - opts[k] = v - for attribute, value in vars(self).items(): - if attribute == "_configs": - continue - opts[attribute] = value - return opts + def __setattr__(self, name: str, value: Any) -> None: + OPTIONS.get()[name] = value + def __getattr__(self, name: str) -> Any: + m = OPTIONS.get() + return m.get(name) + + def set_context(self, options: dict): + """Update current config""" + opts = deepcopy(OPTIONS.get()) + opts.update(options) + OPTIONS.set(opts) + +CONFIG = Config() diff --git a/metagpt/const.py b/metagpt/const.py index 505eebd46..20513461a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : const.py """ +import contextvars from pathlib import Path @@ -35,3 +36,5 @@ TMP = PROJECT_ROOT / 'tmp' RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 + +OPTIONS = contextvars.ContextVar("OPTIONS") From 143ffb0c2cecde75d56d3098044b1cbe1ae5bbe0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 17:45:45 +0800 Subject: [PATCH 113/378] feat: replaced with OPTIONS --- metagpt/actions/action.py | 9 ++-- metagpt/actions/action_output.py | 1 + metagpt/actions/analyze_dep_libs.py | 5 +- metagpt/actions/debug_error.py | 5 +- metagpt/actions/design_api.py | 11 ++--- metagpt/actions/design_api_review.py | 5 +- metagpt/actions/design_filenames.py | 5 +- metagpt/actions/project_management.py | 5 +- metagpt/actions/research.py | 24 ++++----- metagpt/actions/run_code.py | 5 +- metagpt/actions/search_and_summarize.py | 8 +-- metagpt/actions/skill_action.py | 19 ++++++-- metagpt/actions/write_code.py | 5 +- metagpt/actions/write_code_review.py | 5 +- metagpt/actions/write_prd.py | 7 ++- metagpt/actions/write_prd_review.py | 5 +- metagpt/actions/write_teaching_plan.py | 2 +- metagpt/actions/write_test.py | 5 +- metagpt/learn/skill_metadata.py | 25 ---------- metagpt/learn/text_to_embedding.py | 10 +--- metagpt/learn/text_to_image.py | 11 ++--- metagpt/learn/text_to_speech.py | 13 ++--- metagpt/llm.py | 20 ++++++++ metagpt/manager.py | 5 +- metagpt/roles/architect.py | 6 +-- metagpt/roles/customer_service.py | 4 +- metagpt/roles/engineer.py | 6 +-- metagpt/roles/product_manager.py | 6 +-- metagpt/roles/project_manager.py | 6 +-- metagpt/roles/qa_engineer.py | 4 +- metagpt/roles/researcher.py | 9 +--- metagpt/roles/role.py | 59 +++++++---------------- metagpt/roles/sales.py | 4 +- metagpt/roles/seacher.py | 4 +- metagpt/roles/teacher.py | 6 +-- metagpt/software_company.py | 30 +++--------- metagpt/tools/openai_text_to_embedding.py | 6 +-- metagpt/utils/common.py | 15 ------ startup.py | 16 ++---- 39 files changed, 144 insertions(+), 252 deletions(-) delete mode 100644 metagpt/learn/skill_metadata.py create mode 100644 metagpt/llm.py diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 10579d4f4..5cf4f3d81 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -4,7 +4,7 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : action.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. +@Modified By: mashenquan, 2023/8/20. Add function return annotations. """ from abc import ABC from typing import Optional @@ -12,15 +12,16 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.config import Config +from metagpt.llm import LLM from metagpt.utils.common import OutputParser from metagpt.logs import logger class Action(ABC): - def __init__(self, options=None, name: str = '', context=None, llm=None): - self.options = options or Config().runtime_options + def __init__(self, name: str = '', context=None, llm: LLM = None): self.name: str = name + if llm is None: + llm = LLM() self.llm = llm self.context = context self.prefix = "" diff --git a/metagpt/actions/action_output.py b/metagpt/actions/action_output.py index 6c812e7fe..917368798 100644 --- a/metagpt/actions/action_output.py +++ b/metagpt/actions/action_output.py @@ -4,6 +4,7 @@ @Time : 2023/7/11 10:03 @Author : chengmaoyu @File : action_output +@Modified By: mashenquan, 2023/8/20. Allow 'instruct_content' to be blank. """ from typing import Dict, Type, Optional diff --git a/metagpt/actions/analyze_dep_libs.py b/metagpt/actions/analyze_dep_libs.py index d7b251ead..23c35cdf8 100644 --- a/metagpt/actions/analyze_dep_libs.py +++ b/metagpt/actions/analyze_dep_libs.py @@ -4,7 +4,6 @@ @Time : 2023/5/19 12:01 @Author : alexanderwu @File : analyze_dep_libs.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action @@ -27,8 +26,8 @@ Focus only on the names of shared dependencies, do not add any other explanation class AnalyzeDepLibs(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.desc = "根据上下文，分析程序运行依赖库" async def run(self, requirement, filepaths_string): diff --git a/metagpt/actions/debug_error.py b/metagpt/actions/debug_error.py index 78c970337..d69a22dba 100644 --- a/metagpt/actions/debug_error.py +++ b/metagpt/actions/debug_error.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : debug_error.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import re @@ -26,8 +25,8 @@ Now you should start rewriting the code: ## file name of the code to rewrite: Write code with triple quoto. Do your best to implement THIS IN ONLY ONE FILE. """ class DebugError(Action): - def __init__(self, options, name="DebugError", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="DebugError", context=None, llm=None): + super().__init__(name, context, llm) # async def run(self, code, error): # prompt = f"Here is a piece of Python code:\n\n{code}\n\nThe following error occurred during execution:" \ diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index a01e1c753..cf23e6ad1 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : design_api.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import shutil from pathlib import Path @@ -92,8 +91,8 @@ OUTPUT_MAPPING = { class WriteDesign(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ "data structures, library tables, processes, and paths. Please provide your design, feedback " \ "clearly and in detail." @@ -108,15 +107,15 @@ class WriteDesign(Action): def _save_prd(self, docs_path, resources_path, prd): prd_file = docs_path / 'prd.md' quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(options=self.options, mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / 'competitive_analysis') + mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') logger.info(f"Saving PRD to {prd_file}") prd_file.write_text(prd) def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(options=self.options, mermaid_code=data_api_design, output_file_without_suffix=resources_path / 'data_api_design') - mermaid_to_file(options=self.options, mermaid_code=seq_flow, output_file_without_suffix=resources_path / 'seq_flow') + mermaid_to_file(data_api_design, resources_path / 'data_api_design') + mermaid_to_file(seq_flow, resources_path / 'seq_flow') system_design_file = docs_path / 'system_design.md' logger.info(f"Saving System Designs to {system_design_file}") system_design_file.write_text(content) diff --git a/metagpt/actions/design_api_review.py b/metagpt/actions/design_api_review.py index ca4147cca..687a33652 100644 --- a/metagpt/actions/design_api_review.py +++ b/metagpt/actions/design_api_review.py @@ -4,14 +4,13 @@ @Time : 2023/5/11 19:31 @Author : alexanderwu @File : design_api_review.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class DesignReview(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) async def run(self, prd, api_design): prompt = f"Here is the Product Requirement Document (PRD):\n\n{prd}\n\nHere is the list of APIs designed " \ diff --git a/metagpt/actions/design_filenames.py b/metagpt/actions/design_filenames.py index 1f71e9530..6c3d8e803 100644 --- a/metagpt/actions/design_filenames.py +++ b/metagpt/actions/design_filenames.py @@ -4,7 +4,6 @@ @Time : 2023/5/19 11:50 @Author : alexanderwu @File : design_filenames.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import Action from metagpt.logs import logger @@ -16,8 +15,8 @@ Do not add any other explanations, just return a Python string list.""" class DesignFilenames(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.desc = "Based on the PRD, consider system design, and carry out the basic design of the corresponding " \ "APIs, data structures, and database tables. Please give your design, feedback clearly and in detail." diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index d17bf6b03..16473ff01 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -5,7 +5,6 @@ @Author : alexanderwu @File : project_management.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -105,8 +104,8 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, options, name="CreateTasks", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="CreateTasks", context=None, llm=None): + super().__init__(name, context, llm) def _save(self, context, rsp): ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) diff --git a/metagpt/actions/research.py b/metagpt/actions/research.py index 22b0eaa1d..81eb876dd 100644 --- a/metagpt/actions/research.py +++ b/metagpt/actions/research.py @@ -1,9 +1,5 @@ #!/usr/bin/env python -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. -""" - from __future__ import annotations import asyncio @@ -13,6 +9,7 @@ from typing import Callable from pydantic import parse_obj_as from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.tools.search_engine import SearchEngine from metagpt.tools.web_browser_engine import WebBrowserEngine, WebBrowserEngineType @@ -82,15 +79,14 @@ class CollectLinks(Action): """Action class to collect links from a search engine.""" def __init__( self, - options, name: str = "", *args, rank_func: Callable[[list[str]], None] | None = None, **kwargs, ): - super().__init__(options=options, name=name, *args, **kwargs) + super().__init__(name, *args, **kwargs) self.desc = "Collect links from a search engine." - self.search_engine = SearchEngine(options=options) + self.search_engine = SearchEngine() self.rank_func = rank_func async def run( @@ -130,7 +126,7 @@ class CollectLinks(Action): remove.pop() if len(remove) == 0: break - prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, self.options.get("max_tokens_rsp")) + prompt = reduce_message_length(gen_msg(), self.llm.model, system_text, CONFIG.max_tokens_rsp) logger.debug(prompt) queries = await self._aask(prompt, [system_text]) try: @@ -182,10 +178,9 @@ class WebBrowseAndSummarize(Action): **kwargs, ): super().__init__(*args, **kwargs) - if self.options.get("model_for_researcher_summary"): - self.llm.model = self.options.get("model_for_researcher_summary") + if CONFIG.model_for_researcher_summary: + self.llm.model = CONFIG.model_for_researcher_summary self.web_browser_engine = WebBrowserEngine( - options=self.options, engine=WebBrowserEngineType.CUSTOM if browse_func else None, run_func=browse_func, ) @@ -218,8 +213,7 @@ class WebBrowseAndSummarize(Action): for u, content in zip([url, *urls], contents): content = content.inner_text chunk_summaries = [] - for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, - self.options.get("max_tokens_rsp")): + for prompt in generate_prompt_chunk(content, prompt_template, self.llm.model, system_text, CONFIG.max_tokens_rsp): logger.debug(prompt) summary = await self._aask(prompt, [system_text]) if summary == "Not relevant.": @@ -245,8 +239,8 @@ class ConductResearch(Action): """Action class to conduct research and generate a research report.""" def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - if self.options.get("model_for_researcher_report"): - self.llm.model = self.options.get("model_for_researcher_report") + if CONFIG.model_for_researcher_report: + self.llm.model = CONFIG.model_for_researcher_report async def run( self, diff --git a/metagpt/actions/run_code.py b/metagpt/actions/run_code.py index 824ed83fa..f69d2cd1a 100644 --- a/metagpt/actions/run_code.py +++ b/metagpt/actions/run_code.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:46 @Author : alexanderwu @File : run_code.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import os import subprocess @@ -58,8 +57,8 @@ standard errors: {errs}; class RunCode(Action): - def __init__(self, options, name="RunCode", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="RunCode", context=None, llm=None): + super().__init__(name, context, llm) @classmethod async def run_text(cls, code) -> Tuple[str, str]: diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 80d1c52e4..9f54587fa 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -101,16 +101,16 @@ You are a member of a professional butler team and will provide helpful suggesti class SearchAndSummarize(Action): - def __init__(self, options, name="", context=None, llm=None, engine=None, search_func=None): - self.engine = engine or options.get("search_engine") + def __init__(self, name="", context=None, llm=None, engine=None, search_func=None): + self.engine = engine or CONFIG.search_engine try: - self.search_engine = SearchEngine(options=options, engine=self.engine, run_func=search_func) + self.search_engine = SearchEngine(self.engine, run_func=search_func) except pydantic.ValidationError: self.search_engine = None self.result = "" - super().__init__(options=options, name=name, context=context, llm=llm) + super().__init__(name, context, llm) async def run(self, context: list[Message], system_text=SEARCH_AND_SUMMARIZE_SYSTEM) -> str: if self.search_engine is None: diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 8cc7b6c42..c921a5f17 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/28 +@Author : mashenquan +@File : skill_action.py +@Desc : Call learned skill +""" + import ast import importlib @@ -7,8 +16,8 @@ from metagpt.logs import logger class ArgumentsParingAction(Action): - def __init__(self, options, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): - super(ArgumentsParingAction, self).__init__(options=options, name='', context=context, llm=llm) + def __init__(self, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): + super(ArgumentsParingAction, self).__init__(name='', context=context, llm=llm) self.skill = skill self.ask = last_talk self.rsp = None @@ -59,15 +68,15 @@ class ArgumentsParingAction(Action): class SkillAction(Action): - def __init__(self, options, skill: Skill, args: dict, context=None, llm=None, **kwargs): - super(SkillAction, self).__init__(options=options, name='', context=context, llm=llm) + def __init__(self, skill: Skill, args: dict, context=None, llm=None, **kwargs): + super(SkillAction, self).__init__(name='', context=context, llm=llm) self._skill = skill self._args = args self.rsp = None async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" - self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **self.options) + self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **kwargs) return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) @staticmethod diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index 9a2a2f81a..cc122ef7a 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions import WriteDesign from metagpt.actions.action import Action @@ -44,8 +43,8 @@ ATTENTION: Use '##' to SPLIT SECTIONS, not '#'. Output format carefully referenc class WriteCode(Action): - def __init__(self, options, name="WriteCode", context: list[Message] = None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="WriteCode", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) diff --git a/metagpt/actions/write_code_review.py b/metagpt/actions/write_code_review.py index d256c6bcb..7f6a7a38e 100644 --- a/metagpt/actions/write_code_review.py +++ b/metagpt/actions/write_code_review.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_code_review.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action @@ -63,8 +62,8 @@ FORMAT_EXAMPLE = """ class WriteCodeReview(Action): - def __init__(self, options, name="WriteCodeReview", context: list[Message] = None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="WriteCodeReview", context: list[Message] = None, llm=None): + super().__init__(name, context, llm) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 794d3ee9d..0edd24d55 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import List, Tuple @@ -128,11 +127,11 @@ OUTPUT_MAPPING = { class WritePRD(Action): - def __init__(self, options, name="", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="", context=None, llm=None): + super().__init__(name, context, llm) async def run(self, requirements, *args, **kwargs) -> ActionOutput: - sas = SearchAndSummarize(options=self.options, llm=self.llm) + sas = SearchAndSummarize() # rsp = await sas.run(context=requirements, system_text=SEARCH_AND_SUMMARIZE_SYSTEM_EN_US) rsp = "" info = f"### Search Results\n{sas.result}\n\n### Search Summary\n{rsp}" diff --git a/metagpt/actions/write_prd_review.py b/metagpt/actions/write_prd_review.py index 8c22f9c0a..5ff9624c5 100644 --- a/metagpt/actions/write_prd_review.py +++ b/metagpt/actions/write_prd_review.py @@ -4,14 +4,13 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_prd_review.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action class WritePRDReview(Action): - def __init__(self, options, name, context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name, context=None, llm=None): + super().__init__(name, context, llm) self.prd = None self.desc = "Based on the PRD, conduct a PRD Review, providing clear and detailed feedback" self.prd_review_prompt_template = """ diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index 53371b5a1..bd8507350 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -42,7 +42,7 @@ class WriteTeachingPlanPart(Action): statements = [] from metagpt.roles import Role for p in statement_patterns: - s = Role.format_value(p, kwargs) + s = Role.format_value(p) statements.append(s) formatter = self.PROMPT_TITLE_TEMPLATE if self.topic == self.COURSE_TITLE else self.PROMPT_TEMPLATE prompt = formatter.format(formation=self.FORMATION, diff --git a/metagpt/actions/write_test.py b/metagpt/actions/write_test.py index 94006005f..5e50fdb55 100644 --- a/metagpt/actions/write_test.py +++ b/metagpt/actions/write_test.py @@ -4,7 +4,6 @@ @Time : 2023/5/11 17:45 @Author : alexanderwu @File : write_test.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from metagpt.actions.action import Action from metagpt.utils.common import CodeParser @@ -31,8 +30,8 @@ you should correctly import the necessary classes based on these file locations! class WriteTest(Action): - def __init__(self, options, name="WriteTest", context=None, llm=None): - super().__init__(options=options, name=name, context=context, llm=llm) + def __init__(self, name="WriteTest", context=None, llm=None): + super().__init__(name, context, llm) async def write_code(self, prompt): code_rsp = await self._aask(prompt) diff --git a/metagpt/learn/skill_metadata.py b/metagpt/learn/skill_metadata.py deleted file mode 100644 index dea5fb04d..000000000 --- a/metagpt/learn/skill_metadata.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -""" -@Time : 2023/8/20 -@Author : mashenquan -@File : skill_metadata.py -@Desc : Defines metadata for the `skill`. - Depending on the context and specific circumstances, skills may have different effects. - For example: - Proprietor: "Skill of the proprietor entity." - Holder: "Skill of the holder entity." - Possessor: "Skill of the possessor entity." - Controller: "Skill of the controller entity." - Owner: "Skill of the owner entity." -""" - - -def skill_metadata(name, description, requisite): - def decorator(func): - func.skill_name = name - func.skill_description = description - func.skill_requisite = requisite - return func - - return decorator diff --git a/metagpt/learn/text_to_embedding.py b/metagpt/learn/text_to_embedding.py index 5c08ef0b9..26dab0419 100644 --- a/metagpt/learn/text_to_embedding.py +++ b/metagpt/learn/text_to_embedding.py @@ -6,16 +6,11 @@ @File : text_to_embedding.py @Desc : Text-to-Embedding skill, which provides text-to-embedding functionality. """ -import os -from metagpt.learn.skill_metadata import skill_metadata +from metagpt.config import CONFIG from metagpt.tools.openai_text_to_embedding import oas3_openai_text_to_embedding -from metagpt.utils.common import initialize_environment -@skill_metadata(name="Text to Embedding", - description="Convert the text into embeddings.", - requisite="`OPENAI_API_KEY`") async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key="", **kwargs): """Text to embedding @@ -24,7 +19,6 @@ async def text_to_embedding(text, model="text-embedding-ada-002", openai_api_key :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` :return: A json object of :class:`ResultEmbedding` class if successful, otherwise `{}`. """ - initialize_environment() - if os.environ.get("OPENAI_API_KEY") or openai_api_key: + if CONFIG.OPENAI_API_KEY or openai_api_key: return await oas3_openai_text_to_embedding(text, model=model, openai_api_key=openai_api_key) raise EnvironmentError diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index db9844c71..2762c2f18 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -8,15 +8,11 @@ """ import os -from metagpt.learn.skill_metadata import skill_metadata +from metagpt.config import CONFIG from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image -from metagpt.utils.common import initialize_environment -@skill_metadata(name="Text to image", - description="Create a drawing based on the text.", - requisite="`OPENAI_API_KEY` or `METAGPT_TEXT_TO_IMAGE_MODEL`") async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): """Text to image @@ -26,13 +22,12 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod :param model_url: MetaGPT model url :return: The image data is returned in Base64 encoding. """ - initialize_environment() image_declaration = "data:image/png;base64," - if os.environ.get("METAGPT_TEXT_TO_IMAGE_MODEL_URL") or model_url: + if CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL or model_url: data = await oas3_metagpt_text_to_image(text, size_type, model_url) return image_declaration + data if data else "" - if os.environ.get("OPENAI_API_KEY") or openai_api_key: + if CONFIG.OPENAI_API_KEY or openai_api_key: data = await oas3_openai_text_to_image(text, size_type, openai_api_key) return image_declaration + data if data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index e5eb3d488..ba73de04c 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -6,16 +6,14 @@ @File : text_to_speech.py @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ -import os -from metagpt.learn.skill_metadata import skill_metadata + +from metagpt.config import CONFIG + from metagpt.tools.azure_tts import oas3_azsure_tts -from metagpt.utils.common import initialize_environment -@skill_metadata(name="Text to speech", - description="Text-to-speech", - requisite="`AZURE_TTS_SUBSCRIPTION_KEY` and `AZURE_TTS_REGION`") + async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", subscription_key="", region="", **kwargs): """Text to speech @@ -31,9 +29,8 @@ async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style=" :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. """ - initialize_environment() audio_declaration = "data:audio/wav;base64," - if (os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") and os.environ.get("AZURE_TTS_REGION")) or \ + if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or \ (subscription_key and region): data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) return audio_declaration + data if data else data diff --git a/metagpt/llm.py b/metagpt/llm.py new file mode 100644 index 000000000..6a9a9132f --- /dev/null +++ b/metagpt/llm.py @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/5/11 14:45 +@Author : alexanderwu +@File : llm.py +""" + +from metagpt.provider.anthropic_api import Claude2 as Claude +from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + +DEFAULT_LLM = LLM() +CLAUDE_LLM = Claude() + + +async def ai_func(prompt): + """使用LLM进行QA + QA with LLMs + """ + return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/manager.py b/metagpt/manager.py index c4565808e..9d238c621 100644 --- a/metagpt/manager.py +++ b/metagpt/manager.py @@ -4,15 +4,14 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : manager.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ - +from metagpt.llm import LLM from metagpt.logs import logger from metagpt.schema import Message class Manager: - def __init__(self, llm): + def __init__(self, llm: LLM = LLM()): self.llm = llm # Large Language Model self.role_directions = { "BOSS": "Product Manager", diff --git a/metagpt/roles/architect.py b/metagpt/roles/architect.py index 5a498c50b..00b6cb2eb 100644 --- a/metagpt/roles/architect.py +++ b/metagpt/roles/architect.py @@ -4,8 +4,6 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : architect.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WritePRD @@ -14,8 +12,8 @@ from metagpt.roles import Role class Architect(Role): """Architect: Listen to PRD, responsible for designing API, designing code files""" - def __init__(self, options, cost_manager, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", + def __init__(self, name="Bob", profile="Architect", goal="Design a concise, usable, complete python system", constraints="Try to specify good open source tools as much as possible"): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WriteDesign]) self._watch({WritePRD}) diff --git a/metagpt/roles/customer_service.py b/metagpt/roles/customer_service.py index 8550313d4..4aae7cb03 100644 --- a/metagpt/roles/customer_service.py +++ b/metagpt/roles/customer_service.py @@ -26,11 +26,9 @@ DESC = """ class CustomerService(Sales): def __init__( self, - options, - cost_manager, name="Xiaomei", profile="Human customer service", desc=DESC, store=None ): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc, store=store) + super().__init__(name, profile, desc=desc, store=store) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 9da2b5a09..072e53998 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -47,10 +47,10 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, options, cost_manager, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", + def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", n_borg=1, use_code_review=False): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review if self.use_code_review: @@ -131,7 +131,7 @@ class Engineer(Role): async def _act_sp(self) -> Message: code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later for todo in self.todos: - code = await WriteCode(options=self.options, llm=self._llm).run( + code = await WriteCode().run( context=self._rc.history, filename=todo ) diff --git a/metagpt/roles/product_manager.py b/metagpt/roles/product_manager.py index bb69c8dfd..b42e9bb29 100644 --- a/metagpt/roles/product_manager.py +++ b/metagpt/roles/product_manager.py @@ -4,16 +4,14 @@ @Time : 2023/5/11 14:43 @Author : alexanderwu @File : product_manager.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ from metagpt.actions import BossRequirement, WritePRD from metagpt.roles import Role class ProductManager(Role): - def __init__(self, options, cost_manager, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", + def __init__(self, name="Alice", profile="Product Manager", goal="Efficiently create a successful product", constraints=""): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WritePRD]) self._watch([BossRequirement]) diff --git a/metagpt/roles/project_manager.py b/metagpt/roles/project_manager.py index 3e8b36550..ff374de13 100644 --- a/metagpt/roles/project_manager.py +++ b/metagpt/roles/project_manager.py @@ -4,16 +4,14 @@ @Time : 2023/5/11 15:04 @Author : alexanderwu @File : project_manager.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ from metagpt.actions import WriteDesign, WriteTasks from metagpt.roles import Role class ProjectManager(Role): - def __init__(self, options, cost_manager, name="Eve", profile="Project Manager", + def __init__(self, name="Eve", profile="Project Manager", goal="Improve team efficiency and deliver with quality and quantity", constraints=""): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions([WriteTasks]) self._watch([WriteDesign]) diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index ac5df0dbd..65bf2cc5b 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -20,15 +20,13 @@ from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP class QaEngineer(Role): def __init__( self, - options, - cost_manager, name="Edward", profile="QaEngineer", goal="Write comprehensive and robust tests to ensure codes will work as expected without bugs", constraints="The test code you write should conform to code standard like PEP8, be modular, easy to read and maintain", test_round_allowed=5, ): - super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, options=options, cost_manager=cost_manager) + super().__init__(name, profile, goal, constraints) self._init_actions( [WriteTest] ) # FIXME: a bit hack here, only init one action to circumvent _think() logic, will overwrite _think() in future updates diff --git a/metagpt/roles/researcher.py b/metagpt/roles/researcher.py index f3ff7f8e5..cb4d28c33 100644 --- a/metagpt/roles/researcher.py +++ b/metagpt/roles/researcher.py @@ -26,8 +26,6 @@ class Report(BaseModel): class Researcher(Role): def __init__( self, - options, - cost_manager, name: str = "David", profile: str = "Researcher", goal: str = "Gather information and conduct research", @@ -35,11 +33,8 @@ class Researcher(Role): language: str = "en-us", **kwargs, ): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) - self._init_actions([ - CollectLinks(options=options, name=name), - WebBrowseAndSummarize(options=options, name=name), - ConductResearch(options=options, name=name)]) + super().__init__(name, profile, goal, constraints, **kwargs) + self._init_actions([CollectLinks(name), WebBrowseAndSummarize(name), ConductResearch(name)]) self.language = language if language not in ("en-us", "zh-cn"): logger.warning(f"The language `{language}` has not been tested, it may not work.") diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 4f46bb973..a1ac0d9e7 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -4,9 +4,7 @@ @Time : 2023/5/11 14:42 @Author : alexanderwu @File : role.py -@Modified By: mashenquan, 2023-8-7, :class:`Role` + properties. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. +@Modified By: mashenquan, 2023-8-7, Support template-style variables, such as '{teaching_language} Teacher'. @Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. """ from __future__ import annotations @@ -15,7 +13,8 @@ from typing import Iterable, Type, Dict from pydantic import BaseModel, Field -from metagpt.config import Config +from metagpt.config import Config, CONFIG +from metagpt.const import OPTIONS from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager from metagpt.actions import Action, ActionOutput from metagpt.logs import logger @@ -74,13 +73,12 @@ class RoleContext(BaseModel): todo: Action = Field(default=None) watch: set[Type[Action]] = Field(default_factory=set) news: list[Type[Message]] = Field(default=[]) - options: Dict class Config: arbitrary_types_allowed = True def check(self, role_id: str): - if self.options.get("long_term_memory"): + if CONFIG.long_term_memory: self.long_term_memory.recover_memory(role_id, self) self.memory = self.long_term_memory # use memory to act as long_term_memory for unify operation @@ -102,26 +100,20 @@ class RoleContext(BaseModel): class Role: """Role/Proxy""" - def __init__(self, options=None, cost_manager=None, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): - options = options or Config().runtime_options - cost_manager = cost_manager or CostManager(*options) - - self._options = Role.supply_options(options=kwargs, default_options=options) - - name = Role.format_value(name, self._options) - profile = Role.format_value(profile, self._options) - goal = Role.format_value(goal, self._options) - constraints = Role.format_value(constraints, self._options) - desc = Role.format_value(desc, self._options) - - self._cost_manager = cost_manager - self._llm = LLM(options=self._options, cost_manager=cost_manager) + def __init__(self, name="", profile="", goal="", constraints="", desc="", *args, **kwargs): + # Replace template-style variables, such as '{teaching_language} Teacher'. + name = Role.format_value(name) + profile = Role.format_value(profile) + goal = Role.format_value(goal) + constraints = Role.format_value(constraints) + desc = Role.format_value(desc) + self._llm = LLM() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] self._role_id = str(self._setting) - self._rc = RoleContext(options=self._options) + self._rc = RoleContext() def _reset(self): self._states = [] @@ -131,7 +123,7 @@ class Role: self._reset() for idx, action in enumerate(actions): if not isinstance(action, Action): - i = action(options=self._options, name="", llm=self._llm) + i = action("", llm=self._llm) else: i = action i.set_prefix(self._get_prefix(), self.profile) @@ -184,14 +176,6 @@ class Role: """Return number of action""" return len(self._actions) - @property - def options(self): - return self._options - - @options.setter - def options(self, opts): - self._options.update(opts) - def _get_prefix(self): """获取角色前缀""" if self._setting.desc: @@ -222,7 +206,7 @@ class Role: logger.info(f"{self._setting}: ready to {self._rc.todo}") requirement = self._rc.important_memory or self._rc.prerequisite - response = await self._rc.todo.run(requirement, **self._options) + response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): msg = Message(content=response.content, instruct_content=response.instruct_content, @@ -300,23 +284,14 @@ class Role: return rsp @staticmethod - def supply_options(options, default_options=None): - """Supply missing options""" - ret = default_options.copy() if default_options else {} - if not options: - return ret - ret.update(options) - return ret - - @staticmethod - def format_value(value, opts, default_opts=None): + def format_value(value): """Fill parameters inside `value` with `options`.""" if not isinstance(value, str): return value if "{" not in value: return value - merged_opts = Role.supply_options(opts, default_opts) + merged_opts = OPTIONS.get() or {} try: return value.format(**merged_opts) except KeyError as e: diff --git a/metagpt/roles/sales.py b/metagpt/roles/sales.py index 35146fdc3..51b13f487 100644 --- a/metagpt/roles/sales.py +++ b/metagpt/roles/sales.py @@ -13,8 +13,6 @@ from metagpt.tools import SearchEngineType class Sales(Role): def __init__( self, - options, - cost_manager, name="Xiaomei", profile="Retail sales guide", desc="I am a sales guide in retail. My name is Xiaomei. I will answer some customer questions next, and I " @@ -25,7 +23,7 @@ class Sales(Role): "professional guide", store=None ): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, desc=desc) + super().__init__(name, profile, desc=desc) self._set_store(store) def _set_store(self, store): diff --git a/metagpt/roles/seacher.py b/metagpt/roles/seacher.py index 7b07ce713..c116ce98b 100644 --- a/metagpt/roles/seacher.py +++ b/metagpt/roles/seacher.py @@ -13,9 +13,9 @@ from metagpt.tools import SearchEngineType class Searcher(Role): - def __init__(self, options, cost_manager, name='Alice', profile='Smart Assistant', goal='Provide search services for users', + def __init__(self, name='Alice', profile='Smart Assistant', goal='Provide search services for users', constraints='Answer is rich and complete', engine=SearchEngineType.SERPAPI_GOOGLE, **kwargs): - super().__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, goal=goal, constraints=constraints, **kwargs) + super().__init__(name, profile, goal, constraints, **kwargs) self._init_actions([SearchAndSummarize(engine=engine)]) def set_search_func(self, search_func): diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index d2a2198f5..ca88fd681 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -22,13 +22,13 @@ import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, options, name='Lily', profile='{teaching_language} Teacher', + def __init__(self, name='Lily', profile='{teaching_language} Teacher', goal='writing a {language} teaching plan part by part', constraints='writing in {language}', desc="", *args, **kwargs): - super().__init__(options=options, name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: - act = WriteTeachingPlanPart(options=options, topic=topic, llm=self._llm) + act = WriteTeachingPlanPart(topic=topic, llm=self._llm) actions.append(act) self._init_actions(actions) self._watch({TeachingPlanRequirement}) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 529dc0fe7..8f173ebf3 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -4,22 +4,16 @@ @Time : 2023/5/12 00:30 @Author : alexanderwu @File : software_company.py -@Modified By: mashenquan, 2023-07-27, Add `role` & `cause_by` parameters to `start_project()`. -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ -from typing import Dict - from pydantic import BaseModel, Field from metagpt.actions import BossRequirement +from metagpt.config import CONFIG from metagpt.environment import Environment from metagpt.logs import logger -from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message from metagpt.utils.common import NoMoneyException -from metagpt.config import Config class SoftwareCompany(BaseModel): @@ -30,8 +24,6 @@ class SoftwareCompany(BaseModel): environment: Environment = Field(default_factory=Environment) investment: float = Field(default=10.0) idea: str = Field(default="") - options: Dict = Field(default=Config().runtime_options) - cost_manager: CostManager = Field(default=CostManager(**Config().runtime_options)) class Config: arbitrary_types_allowed = True @@ -43,17 +35,17 @@ class SoftwareCompany(BaseModel): def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment - self.options["max_budget"] = investment + CONFIG.max_budget = investment logger.info(f'Investment: ${investment}.') def _check_balance(self): - if self.total_cost > self.max_budget: - raise NoMoneyException(self.total_cost, f'Insufficient funds: {self.max_budget}') + if CONFIG.total_cost > CONFIG.max_budget: + raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea, role="BOSS", cause_by=BossRequirement): + def start_project(self, idea): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role=role, content=idea, cause_by=cause_by)) + self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) def _save(self): logger.info(self.json()) @@ -67,13 +59,3 @@ class SoftwareCompany(BaseModel): self._check_balance() await self.environment.run() return self.environment.history - - @property - def max_budget(self): - return self.options.get("max_budget", 0) - - @property - def total_cost(self): - return self.options.get("total_cost", 0) - - diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 119eb35b6..73984aff6 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -17,8 +17,9 @@ import requests from pydantic import BaseModel import sys +from metagpt.config import CONFIG + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -83,12 +84,11 @@ async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", op if not text: return "" if not openai_api_key: - openai_api_key = os.environ.get("OPENAI_API_KEY") + openai_api_key = CONFIG.OPENAI_API_KEY return await OpenAIText2Embedding(openai_api_key).text_2_embedding(text, model=model) if __name__ == "__main__": - initialize_environment() loop = asyncio.new_event_loop() v = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) loop.run_until_complete(v) diff --git a/metagpt/utils/common.py b/metagpt/utils/common.py index a6e4dc20d..791bb2767 100644 --- a/metagpt/utils/common.py +++ b/metagpt/utils/common.py @@ -259,18 +259,3 @@ def parse_recipient(text): recipient = re.search(pattern, text) return recipient.group(1) if recipient else "" - -def initialize_environment(options=None): - """Load `config/config.yaml` to `os.environ`""" - if options: - for k, v in options.items(): - os.environ[k] = str(v) - return - - yaml_file_path = Path(__file__).resolve().parent.parent.parent / "config/config.yaml" - if not yaml_file_path.exists(): - return - with open(str(yaml_file_path), "r") as yaml_file: - data = yaml.safe_load(yaml_file) - for k, v in data.items(): - os.environ[k] = str(v) diff --git a/startup.py b/startup.py index 84cd43956..03b2149c4 100644 --- a/startup.py +++ b/startup.py @@ -1,10 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. -""" - import asyncio import platform import fire @@ -16,15 +11,14 @@ from metagpt.software_company import SoftwareCompany async def startup(idea: str, investment: float = 3.0, n_round: int = 5, code_review: bool = False, run_tests: bool = False): """Run a startup. Be a boss.""" - company = SoftwareCompany() - company.hire([ProductManager(options=company.options, cost_manager=company.cost_manager), - Architect(options=company.options, cost_manager=company.cost_manager), - ProjectManager(options=company.options, cost_manager=company.cost_manager), - Engineer(n_borg=5, use_code_review=code_review, options=company.options, cost_manager=company.cost_manager)]) + company.hire([ProductManager(), + Architect(), + ProjectManager(), + Engineer(n_borg=5, use_code_review=code_review)]) if run_tests: # developing features: run tests on the spot and identify bugs (bug fixing capability comes soon!) - company.hire([QaEngineer(options=company.options, cost_manager=company.cost_manager)]) + company.hire([QaEngineer()]) company.invest(investment) company.start_project(idea) await company.run(n_round=n_round) From 23ba0f3540c90f0e3336741c98ad50debcd0d6c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 17:56:50 +0800 Subject: [PATCH 114/378] feat: replaced with OPTIONS --- metagpt/provider/anthropic_api.py | 15 ++----- metagpt/provider/openai_api.py | 65 +++++++++++-------------------- 2 files changed, 26 insertions(+), 54 deletions(-) diff --git a/metagpt/provider/anthropic_api.py b/metagpt/provider/anthropic_api.py index 326d23a5c..03802a716 100644 --- a/metagpt/provider/anthropic_api.py +++ b/metagpt/provider/anthropic_api.py @@ -4,22 +4,17 @@ @Time : 2023/7/21 11:15 @Author : Leo Xiao @File : anthropic_api.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation; - Change cost control from global to company level. """ import anthropic from anthropic import Anthropic -from metagpt.config import Config +from metagpt.config import CONFIG class Claude2: - def __init__(self, options=None): - self.options = options or Config().runtime_options - def ask(self, prompt): - client = Anthropic(api_key=self.claude_api_key) + client = Anthropic(api_key=CONFIG.claude_api_key) res = client.completions.create( model="claude-2", @@ -29,7 +24,7 @@ class Claude2: return res.completion async def aask(self, prompt): - client = Anthropic(api_key=self.claude_api_key) + client = Anthropic(api_key=CONFIG.claude_api_key) res = client.completions.create( model="claude-2", @@ -37,7 +32,3 @@ class Claude2: max_tokens_to_sample=1000, ) return res.completion - - @property - def claude_api_key(self): - return self.options.get("claude_api_key") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 098388a7c..640694b67 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -18,6 +18,7 @@ from openai.error import APIConnectionError from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.token_counter import ( @@ -134,23 +135,22 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self, options, cost_manager): - self._options = options - self.__init_openai() + def __init__(self, cost_manager): + self.__init_openai(CONFIG) self.llm = openai - self.model = self.openai_api_model + self.model = CONFIG.openai_api_model self.auto_max_tokens = False - self._cost_manager = cost_manager + self._cost_manager = cost_manager or CostManager() RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self): - openai.api_key = self.openai_api_key - if self.openai_api_base: - openai.api_base = self.openai_api_base - if self.openai_api_type: - openai.api_type = self.openai_api_type - openai.api_version = self.openai_api_version - self.rpm = int(self._options.get("RPM", 10)) + def __init_openai(self, config): + openai.api_key = config.openai_api_key + if config.openai_api_base: + openai.api_base = config.openai_api_base + if config.openai_api_type: + openai.api_type = config.openai_api_type + openai.api_version = config.openai_api_version + self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await self.async_retry_call(openai.ChatCompletion.acreate, @@ -175,9 +175,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return full_reply_content def _cons_kwargs(self, messages: list[dict]) -> dict: - if self._options.get("openai_api_type") == "azure": + if CONFIG.openai_api_type == "azure": kwargs = { - "deployment_id": self._options.get("deployment_id"), + "deployment_id": CONFIG.deployment_id, "messages": messages, "max_tokens": self.get_max_tokens(messages), "n": 1, @@ -232,7 +232,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _calc_usage(self, messages: list[dict], rsp: str) -> dict: usage = {} - if self._options.get("calc_usage"): + if CONFIG.calc_usage: try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) @@ -271,7 +271,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return results def _update_costs(self, usage: dict): - if self._options.get("calc_usage"): + if CONFIG.calc_usage: try: prompt_tokens = int(usage['prompt_tokens']) completion_tokens = int(usage['completion_tokens']) @@ -284,34 +284,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def get_max_tokens(self, messages: list[dict]): if not self.auto_max_tokens: - return self._options.get("max_tokens_rsp") - return get_max_completion_tokens(messages, self.model, self._options.get("max_tokens_rsp")) - - @property - def openai_api_model(self): - return self._options.get("openai_api_model") - - @property - def openai_api_key(self): - return self._options.get("openai_api_key") - - @property - def openai_api_base(self): - return self._options.get("openai_api_base") - - @property - def openai_api_type(self): - return self._options.get("openai_api_type") - - @property - def openai_api_version(self): - return self._options.get("openai_api_version") + return CONFIG.max_tokens_rsp + return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) async def get_summary(self, text: str, max_words=20): """Generate text summary""" if len(text) < max_words: return text - language = self._options.get("language", "English") + language = CONFIG.language or self.DEFAULT_LANGUAGE command = f"Translate the above content into a {language} summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") @@ -322,7 +302,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: """Generate text title""" max_response_token_count = 50 - max_token_count = max_token_count_per_ask or self._options.get("MAX_TOKENS", 1500) + max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or 1500 text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) summaries = [] @@ -332,7 +312,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(summaries) == 1: return summaries[0] - language = self._options.get("language", "English") + language = CONFIG.language or self.DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." summaries.append(command) msg = "\n".join(summaries) @@ -418,3 +398,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + DEFAULT_LANGUAGE = "Engilish" From 7895af2c5a59511c3ba01e50420890a1cd85460b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:07:59 +0800 Subject: [PATCH 115/378] feat: replace CONFIG with OPTIONS --- examples/write_teaching_plan.py | 4 ++-- metagpt/actions/search_and_summarize.py | 1 + metagpt/actions/write_teaching_plan.py | 4 ++-- metagpt/config.py | 5 +++++ metagpt/provider/openai_api.py | 4 ++-- metagpt/roles/assistant.py | 7 ++++--- metagpt/software_company.py | 4 ++-- metagpt/tools/search_engine.py | 12 ++++++------ 8 files changed, 24 insertions(+), 17 deletions(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 6ab5edce4..2a9c4c0e5 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -77,9 +77,9 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * lesson = demo_lesson company = SoftwareCompany() - company.hire([Teacher(options=company.options, cost_manager=company.cost_manager, *args, **kwargs)]) + company.hire([Teacher(*args, **kwargs)]) company.invest(investment) - company.start_project(lesson, role="Teacher", cause_by=TeachingPlanRequirement) + company.start_project(lesson, cause_by=TeachingPlanRequirement, role="Teacher", **kwargs) await company.run(n_round=1) diff --git a/metagpt/actions/search_and_summarize.py b/metagpt/actions/search_and_summarize.py index 9f54587fa..5c7577e17 100644 --- a/metagpt/actions/search_and_summarize.py +++ b/metagpt/actions/search_and_summarize.py @@ -9,6 +9,7 @@ import pydantic from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.schema import Message from metagpt.tools.search_engine import SearchEngine diff --git a/metagpt/actions/write_teaching_plan.py b/metagpt/actions/write_teaching_plan.py index bd8507350..7c959ce85 100644 --- a/metagpt/actions/write_teaching_plan.py +++ b/metagpt/actions/write_teaching_plan.py @@ -20,7 +20,7 @@ class TeachingPlanRequirement(Action): class WriteTeachingPlanPart(Action): """Write Teaching Plan Part""" - def __init__(self, options, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): + def __init__(self, name: str = "", context=None, llm=None, topic: str = "", language: str = "Chinese"): """ :param name: action name @@ -29,7 +29,7 @@ class WriteTeachingPlanPart(Action): :param topic: topic part of teaching plan :param language: A human language, such as Chinese, English, French, etc. """ - super().__init__(options, name, context, llm) + super().__init__(name, context, llm) self.topic = topic self.language = language self.rsp = None diff --git a/metagpt/config.py b/metagpt/config.py index ceaa582e2..a3edc22b6 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -126,4 +126,9 @@ class Config(metaclass=Singleton): opts.update(options) OPTIONS.set(opts) + @property + def options(self): + """Return all key-values""" + return OPTIONS.get() + CONFIG = Config() diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 640694b67..02bf5126c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -67,7 +67,7 @@ class CostManager(BaseModel): total_prompt_tokens: int = 0 total_completion_tokens: int = 0 total_budget: float = 0 - max_budget: float + max_budget: float = CONFIG.max_budget total_cost: float = 0 def update_cost(self, prompt_tokens, completion_tokens, model): @@ -135,7 +135,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self, cost_manager): + def __init__(self, cost_manager=None): self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index d6f52e4e4..c8a786b41 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -10,7 +10,8 @@ For more about `fork` node in activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` This file defines a `fork` style meta role capable of generating arbitrary roles at runtime based on a configuration file. -@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false indicates that further reasoning cannot continue. +@Modified By: mashenquan, 2023/8/22. A definition has been provided for the return value of _think: returning false + indicates that further reasoning cannot continue. """ import asyncio @@ -34,7 +35,7 @@ SKILL_PATH = "SKILL_PATH" class Assistant(Role): - """解决通用问题的助手""" + """Assistant for solving common issues.""" def __init__(self, options, cost_manager, name="Lily", profile="An assistant", goal="Help to solve problem", constraints="Talk in {language}", desc="", *args, **kwargs): @@ -152,7 +153,7 @@ async def main(): break msg = await role.act() logger.info(msg) - # 获取用户终端输入 + # Retrieve user terminal input. logger.info("Enter prompt") talk = input("You: ") await role.talk(talk) diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8f173ebf3..8d9c990ee 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -42,10 +42,10 @@ class SoftwareCompany(BaseModel): if CONFIG.total_cost > CONFIG.max_budget: raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') - def start_project(self, idea): + def start_project(self, idea, role="BOSS", cause_by=BossRequirement, **kwargs): """Start a project from publishing boss requirement.""" self.idea = idea - self.environment.publish_message(Message(role="BOSS", content=idea, cause_by=BossRequirement)) + self.environment.publish_message(Message(content=idea, role=role, cause_by=cause_by)) def _save(self): logger.info(self.json()) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index c82ae6595..5b8b7f046 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -11,6 +11,7 @@ from __future__ import annotations import importlib from typing import Callable, Coroutine, Literal, overload, Dict +from metagpt.config import CONFIG from metagpt.tools import SearchEngineType @@ -28,23 +29,22 @@ class SearchEngine: def __init__( self, - options: Dict, engine: SearchEngineType | None = None, run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None ): - engine = engine or options.get("search_engine") + engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper(**options).run + run_func = importlib.import_module(module).SerpAPIWrapper(**CONFIG.options).run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" - run_func = importlib.import_module(module).SerperWrapper(**options).run + run_func = importlib.import_module(module).SerperWrapper(**CONFIG.options).run elif engine == SearchEngineType.DIRECT_GOOGLE: module = "metagpt.tools.search_engine_googleapi" - run_func = importlib.import_module(module).GoogleAPIWrapper(**options).run + run_func = importlib.import_module(module).GoogleAPIWrapper(**CONFIG.options).run elif engine == SearchEngineType.DUCK_DUCK_GO: module = "metagpt.tools.search_engine_ddg" - run_func = importlib.import_module(module).DDGAPIWrapper(**options).run + run_func = importlib.import_module(module).DDGAPIWrapper(**CONFIG.options).run elif engine == SearchEngineType.CUSTOM_ENGINE: pass # run_func = run_func else: From 00f1e1882036261b16f1fb682153bc71f0059edd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:13:41 +0800 Subject: [PATCH 116/378] feat: replace CONFIG with OPTIONS --- examples/write_teaching_plan.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 2a9c4c0e5..191547193 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -11,6 +11,8 @@ import asyncio from pathlib import Path import sys +from metagpt.config import CONFIG + sys.path.append(str(Path(__file__).resolve().parent.parent)) import aiofiles import fire @@ -66,6 +68,7 @@ async def startup(lesson_file: str, investment: float = 3.0, n_round: int = 1, * 3c Match the big letters with the small ones. Then write them on the lines. """ + CONFIG.set_context(kwargs) lesson = "" if lesson_file and Path(lesson_file).exists(): From 3a96405a692efdd7ca96b104c73983744ce48a46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:15:48 +0800 Subject: [PATCH 117/378] feat: delete useless config --- config/pattern/template.yaml | 40 -------- config/pattern/write_teaching_plan.yaml | 126 ------------------------ 2 files changed, 166 deletions(-) delete mode 100644 config/pattern/template.yaml delete mode 100644 config/pattern/write_teaching_plan.yaml diff --git a/config/pattern/template.yaml b/config/pattern/template.yaml deleted file mode 100644 index d148804f0..000000000 --- a/config/pattern/template.yaml +++ /dev/null @@ -1,40 +0,0 @@ -# Pattern Configuration Template -# Created By: mashenquan, 2023-8-7 -# File Name: template.yaml -# This template defines a set of structural standards for generating roles and action flows based on configurations. -# For more about UML 2.0 activity diagrams, see: `https://www.uml-diagrams.org/activity-diagrams.html` - -# project settings -startup: - requirement: "TeachingPlanRequirement" # Defines project initial requirement action - role: "Teacher" # Defines project role - investment: 3.0 # Defines the max project investment - n_round: 1 # Defines the max project round count - -# roles settings -roles: # A project can involve multiple roles. -- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. - name: "Lily" - profile: "{teaching_language} Teacher" - goal: "writing a {language} teaching plan part by part" - constraints: "writing in {language}" - role: "You are a {teaching_language} Teacher, named Lily, your goal is ..." - desc: "" - output_filename: "teaching_plan_demo.md" - requirement: ["TeachingPlanRequirement"] - templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. - - "Do ..." - - "Do ..." - # role's action settings - actions: # A role can have multiple actions. - - name: "" - topic: "Title" - language: "Chinese" - statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. - - "Statement: Find and return ..." - template_ix: 0 - rsp_begin_tag: "[..._BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. - rsp_end_tag: "[..._END]" # When asking, request the LLM to include the tag in the response. It's optional. - - - diff --git a/config/pattern/write_teaching_plan.yaml b/config/pattern/write_teaching_plan.yaml deleted file mode 100644 index 5b5f2af77..000000000 --- a/config/pattern/write_teaching_plan.yaml +++ /dev/null @@ -1,126 +0,0 @@ -# The `fork` role demo implements the flow of the code in `examples/write_teaching_plan.py`. - -# project settings -startup: - requirement: "TeachingPlanRequirement" # Defines project initial requirement action - role: "Teacher" - investment: 3.0 - n_round: 1 - -# roles settings -roles: # A project can involve multiple roles. -- role_type: "fork" # `fork` type role corresponds to the functional positioning of the `fork` node in UML 2.0 activity diagrams. - name: "Lily" - profile: "{teaching_language} Teacher" - goal: "writing a {language} teaching plan part by part" - constraints: "writing in {language}" - role: "You are a {teaching_language} Teacher, named Lily, your goal is writing a {teaching_language} teaching plan part by part, and the constraint is writing in {language}." - desc: "" - output_filename: "teaching_plan_demo" - requirement: ["TeachingPlanRequirement"] - templates: # The template provides a convenient way to generate prompts. After each action selects its respective template, you only need to provide the corresponding variable values. Variable replacement is automatically handled by the framework. - - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\n{statements}\nConstraint: Writing in {language}.\nAnswer options: Encloses the lesson title with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" - - "Do not refer to the context of the previous conversation records, start the conversation anew.\n\nFormation: \"Capacity and role\" defines the role you are currently playing;\n\t\"[LESSON_BEGIN]\" and \"[LESSON_END]\" tags enclose the content of textbook;\n\t\"Statement\" defines the work detail you need to complete at this stage;\n\t\"Answer options\" defines the format requirements for your responses;\n\t\"Constraint\" defines the conditions that your responses must comply with.\n\nCapacity and role: {role}\nStatement: Write the \"{topic}\" part of teaching plan, WITHOUT ANY content unrelated to \"{topic}\"!!\n{statements}\nAnswer options: Enclose the teaching plan content with \"[TEACHING_PLAN_BEGIN]\" and \"[TEACHING_PLAN_END]\" tags.\nAnswer options: Using proper markdown format from second-level header format.\nConstraint: Writing in {language}.\n[LESSON_BEGIN]\n{lesson}\n[LESSON_END]" - actions: # 一个role可以有多个action - - name: "" - topic: "Title" - language: "Chinese" - statements: # When replacing template variables, multiple statements will be joined into a single string using line breaks. - - "Statement: Find and return the title of the lesson only with \"# \" string prefixed, without anything else." - template_ix: 0 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Hours" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" # When asking, request the LLM to include the tag in the response. It's optional. - rsp_end_tag: "[TEACHING_PLAN_END]" # When asking, request the LLM to include the tag in the response. It's optional. - - name: "" - topic: "Teaching Objectives" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Content" - language: "Chinese" - statements: - - "Statement: \"Teaching Content\" must include vocabulary, analysis, and examples of various grammar structures that appear in the textbook, as well as the listening materials and key points." - - "Statement: \"Teaching Content\" must include more examples." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Methods and Strategies" - language: "Chinese" - statements: - - "Statement: \"Teaching Methods and Strategies\" must include teaching focus, difficulties, materials, procedures, in detail." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Learning Activities" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Time Allocation" - language: "Chinese" - statements: - - "Statement: \"Teaching Time Allocation\" must include how much time is allocated to each part of the textbook content." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Assessment and Feedback" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Teaching Summary and Improvement" - language: "Chinese" - statements: [] - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Vocabulary Cloze" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create vocabulary cloze. The cloze should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers. The key-related vocabulary and phrases in the textbook content must all be included in the exercises." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Choice Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create choice questions. 10 questions." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Grammar Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create grammar questions. 10 questions." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - name: "" - topic: "Translation Questions" - language: "Chinese" - statements: - - "Statement: Based on the content of the textbook enclosed by \"[LESSON_BEGIN]\" and \"[LESSON_END]\", create translation questions. The translation should include 10 {language} questions with {teaching_language} answers, and it should also include 10 {teaching_language} questions with {language} answers." - template_ix: 1 - rsp_begin_tag: "[TEACHING_PLAN_BEGIN]" - rsp_end_tag: "[TEACHING_PLAN_END]" - - From 3243078b77d15874a2fde38a2833005ebe0d143e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:21:50 +0800 Subject: [PATCH 118/378] feat: replace CONFIG with OPTIONS --- metagpt/actions/talk_action.py | 19 ++++++++++++++----- metagpt/const.py | 1 + metagpt/provider/openai_api.py | 7 ++++--- 3 files changed, 19 insertions(+), 8 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 5692cf4f4..555b202d1 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -1,15 +1,25 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/28 +@Author : mashenquan +@File : talk_action.py +@Desc : Act as it’s a talk +""" + from metagpt.actions import Action, ActionOutput +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_LANGUAGE from metagpt.logs import logger - class TalkAction(Action): - def __init__(self, options, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): + def __init__(self, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary context["knowledge"] = knowledge - super(TalkAction, self).__init__(options=options, name=name, context=context, llm=llm) + super(TalkAction, self).__init__(name=name, context=context, llm=llm) self._talk = talk self._history_summary = history_summary self._knowledge = knowledge @@ -21,7 +31,7 @@ class TalkAction(Action): prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " - language = self.options.get("language", "Chinese") + language = CONFIG.language or DEFAULT_LANGUAGE prompt += f"Answer in {language}:\n {self._talk}" return prompt @@ -32,4 +42,3 @@ class TalkAction(Action): logger.info(rsp) self._rsp = ActionOutput(content=rsp) return self._rsp - diff --git a/metagpt/const.py b/metagpt/const.py index 20513461a..0e50f2c39 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -38,3 +38,4 @@ RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") +DEFAULT_LANGUAGE = "Engilish" \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 02bf5126c..45e67739b 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -19,6 +19,7 @@ from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG +from metagpt.const import DEFAULT_LANGUAGE from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.token_counter import ( @@ -291,7 +292,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """Generate text summary""" if len(text) < max_words: return text - language = CONFIG.language or self.DEFAULT_LANGUAGE + language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above content into a {language} summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") @@ -312,7 +313,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(summaries) == 1: return summaries[0] - language = CONFIG.language or self.DEFAULT_LANGUAGE + language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." summaries.append(command) msg = "\n".join(summaries) @@ -398,4 +399,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_LANGUAGE = "Engilish" + From 7c4b5b40828918d3084ac622bb4293d1ac8c0a4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:26:21 +0800 Subject: [PATCH 119/378] feat: fix coding --- metagpt/const.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 0e50f2c39..a14dbc5b8 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -3,7 +3,8 @@ """ @Time : 2023/5/1 11:59 @Author : alexanderwu -@File : const.py +@File : const.py' +@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE' """ import contextvars from pathlib import Path @@ -38,4 +39,4 @@ RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") -DEFAULT_LANGUAGE = "Engilish" \ No newline at end of file +DEFAULT_LANGUAGE = "English" From 1c2b14b46df1f28f7131acc71ab5887ff69e690b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:31:07 +0800 Subject: [PATCH 120/378] feat: + annotations --- metagpt/learn/skill_loader.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 71535f310..cbf63c60a 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : skill_loader.py +@Desc : Skill YAML Configuration Loader. +""" + from pathlib import Path from typing import List, Dict, Optional @@ -9,10 +18,12 @@ class Example(BaseModel): ask: str answer: str + class Returns(BaseModel): type: str format: Optional[str] = None + class Skill(BaseModel): name: str description: str @@ -40,6 +51,7 @@ class SkillLoader: self._skills = SkillsDeclaration(**skills) def get_skill_list(self, entity_name: str = "Assistant") -> Dict: + """Return the skill name based on the skill description.""" entity_skills = self.get_entity(entity_name) if not entity_skills: return {} @@ -51,6 +63,7 @@ class SkillLoader: return description_to_name_mappings def get_skill(self, name, entity_name: str = "Assistant") -> Skill: + """Return a skill by name.""" entity = self.get_entity(entity_name) if not entity: return None @@ -59,6 +72,7 @@ class SkillLoader: return sk def get_entity(self, name) -> EntitySkills: + """Return a list of skills for the entity.""" if not self._skills: return None - return self._skills.entities.get(name) \ No newline at end of file + return self._skills.entities.get(name) From deccb9fde272312c0e5de2fe5262bc7db5d8f802 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:35:30 +0800 Subject: [PATCH 121/378] feat: + annotations --- metagpt/config.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metagpt/config.py b/metagpt/config.py index a3edc22b6..f1c869b6c 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -52,10 +52,12 @@ class Config(metaclass=Singleton): ): raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") - openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy - if openai_proxy: - openai.proxy = openai_proxy - openai.api_base = self.openai_api_base + if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: + openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy + if openai_proxy: + openai.proxy = openai_proxy + else: + logger.info("Set OPENAI_API_BASE in case of network issues") self.openai_api_type = self._get("OPENAI_API_TYPE") self.openai_api_version = self._get("OPENAI_API_VERSION") self.openai_api_rpm = self._get("RPM", 3) From 8738831e0fdfcfd2a6f60c30bf419b3130241232 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:38:27 +0800 Subject: [PATCH 122/378] feat: + annotations --- metagpt/learn/text_to_image.py | 1 - metagpt/learn/text_to_speech.py | 4 +--- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 2762c2f18..620e58180 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,7 +6,6 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ -import os from metagpt.config import CONFIG from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index ba73de04c..66fbba5be 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -7,15 +7,13 @@ @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ - from metagpt.config import CONFIG from metagpt.tools.azure_tts import oas3_azsure_tts - async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", - subscription_key="", region="", **kwargs): + subscription_key="", region="", **kwargs): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` From 455c59d8c4af7abeb8c080bdc167e2369e00c6f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:41:32 +0800 Subject: [PATCH 123/378] feat: + annotations --- metagpt/memory/brain_memory.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index cb67fea8e..b3445a1f2 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -1,3 +1,12 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/18 +@Author : mashenquan +@File : brain_memory.py +@Desc : Support memory for multiple tasks and multiple mainlines. +""" + from enum import Enum from typing import List, Dict From 27561765cf49c421147fd4e4bf2d76a37672aa5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:45:56 +0800 Subject: [PATCH 124/378] feat: + annotations --- metagpt/const.py | 3 ++- metagpt/provider/openai_api.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index a14dbc5b8..8c1460a02 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,7 +4,7 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py' -@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE' +@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE', 'DEFAULT_MAX_TOKENS' """ import contextvars from pathlib import Path @@ -40,3 +40,4 @@ MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") DEFAULT_LANGUAGE = "English" +DEFAULT_MAX_TOKENS = 1500 \ No newline at end of file diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 45e67739b..7dba00530 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -19,7 +19,7 @@ from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG -from metagpt.const import DEFAULT_LANGUAGE +from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.token_counter import ( @@ -303,7 +303,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: """Generate text title""" max_response_token_count = 50 - max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or 1500 + max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or DEFAULT_MAX_TOKENS text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) summaries = [] From 946e6fa8b39d82f5f688c01bdcd4f3a1e20d1464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 19:47:57 +0800 Subject: [PATCH 125/378] feat: + annotations --- metagpt/const.py | 7 +++++-- metagpt/roles/assistant.py | 6 +----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 8c1460a02..9e7462da6 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -4,7 +4,7 @@ @Time : 2023/5/1 11:59 @Author : alexanderwu @File : const.py' -@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE', 'DEFAULT_MAX_TOKENS' +@Modified By: mashenquan, 2023/8/28. Add 'OPTIONS', 'DEFAULT_LANGUAGE', 'DEFAULT_MAX_TOKENS'... """ import contextvars from pathlib import Path @@ -40,4 +40,7 @@ MEM_TTL = 24 * 30 * 3600 OPTIONS = contextvars.ContextVar("OPTIONS") DEFAULT_LANGUAGE = "English" -DEFAULT_MAX_TOKENS = 1500 \ No newline at end of file +DEFAULT_MAX_TOKENS = 1500 +COMMAND_TOKENS = 500 +BRAIN_MEMORY = "BRAIN_MEMORY" +SKILL_PATH = "SKILL_PATH" \ No newline at end of file diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c8a786b41..7d1517d7e 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -21,6 +21,7 @@ from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction from metagpt.config import Config +from metagpt.const import BRAIN_MEMORY, SKILL_PATH from metagpt.learn.skill_loader import SkillLoader from metagpt.logs import logger from metagpt.memory.brain_memory import BrainMemory, MessageType @@ -28,11 +29,6 @@ from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message -DEFAULT_MAX_TOKENS = 1500 -COMMAND_TOKENS = 500 -BRAIN_MEMORY = "BRAIN_MEMORY" -SKILL_PATH = "SKILL_PATH" - class Assistant(Role): """Assistant for solving common issues.""" From 71b4922f554a0dc411bde745066ecc286ad0fc5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 20:17:55 +0800 Subject: [PATCH 126/378] feat: fix coding --- metagpt/learn/skill_loader.py | 12 ++++++++--- metagpt/roles/assistant.py | 29 +++++++++++++------------- metagpt/tools/azure_tts.py | 3 --- metagpt/tools/metagpt_oas3_api_svc.py | 3 --- metagpt/tools/metagpt_text_to_image.py | 3 --- metagpt/tools/openai_text_to_image.py | 3 --- tests/metagpt/tools/test_azure_tts.py | 3 --- 7 files changed, 23 insertions(+), 33 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index cbf63c60a..1cd83240d 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -6,12 +6,11 @@ @File : skill_loader.py @Desc : Skill YAML Configuration Loader. """ - from pathlib import Path from typing import List, Dict, Optional import yaml -from pydantic import BaseModel +from pydantic import BaseModel, Field class Example(BaseModel): @@ -24,11 +23,18 @@ class Returns(BaseModel): format: Optional[str] = None +class Prerequisite(BaseModel): + name: str + type: Optional[str] = None + description: Optional[str] = None + default: Optional[str] = None + + class Skill(BaseModel): name: str description: str id: str - requisite: List[str] + x_prerequisite: Optional[List[Prerequisite]] = Field(default=None, alias="x-prerequisite") arguments: Dict examples: List[Example] returns: Returns diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 7d1517d7e..944b250f1 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -20,7 +20,7 @@ from pathlib import Path from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction -from metagpt.config import Config +from metagpt.config import Config, CONFIG from metagpt.const import BRAIN_MEMORY, SKILL_PATH from metagpt.learn.skill_loader import SkillLoader from metagpt.logs import logger @@ -33,13 +33,13 @@ from metagpt.schema import Message class Assistant(Role): """Assistant for solving common issues.""" - def __init__(self, options, cost_manager, name="Lily", profile="An assistant", goal="Help to solve problem", + def __init__(self, name="Lily", profile="An assistant", goal="Help to solve problem", constraints="Talk in {language}", desc="", *args, **kwargs): - super(Assistant, self).__init__(options=options, cost_manager=cost_manager, name=name, profile=profile, + super(Assistant, self).__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) - brain_memory = options.get(BRAIN_MEMORY) + brain_memory = CONFIG.BRAIN_MEMORY self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() - skill_path = Path(options.get(SKILL_PATH)) if options.get(SKILL_PATH) else None + skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None self.skills = SkillLoader(skill_yaml_file_name=skill_path) async def think(self) -> bool: @@ -60,7 +60,7 @@ class Assistant(Role): return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: - result = await self._rc.todo.run(**self._options) + result = await self._rc.todo.run(**CONFIG.options) if not result: return None if isinstance(result, str): @@ -87,7 +87,7 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(options=self.options, talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, + action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, **kwargs) self.add_to_do(action) return True @@ -98,12 +98,11 @@ class Assistant(Role): if not skill: logger.info(f"skill not found: {text}") return await self.talk_handler(text=last_talk, **kwargs) - action = ArgumentsParingAction(options=self.options, skill=skill, llm=self._llm, **kwargs) + action = ArgumentsParingAction(skill=skill, llm=self._llm, **kwargs) await action.run(**kwargs) if action.args is None: return await self.talk_handler(text=last_talk, **kwargs) - action = SkillAction(options=self.options, skill=skill, args=action.args, llm=self._llm, name=skill.name, - desc=skill.description) + action = SkillAction(skill=skill, args=action.args, llm=self._llm, name=skill.name, desc=skill.description) self.add_to_do(action) return True @@ -115,11 +114,11 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_context_title(history_text, max_words=20) - if last_talk and await self._llm.is_related(last_talk, history_summary): # 合并相关内容 + if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk - self.memory.move_to_solution() # 问题解决后及时清空内存 + self.memory.move_to_solution() # Promptly clear memory after the issue is resolved. return last_talk @staticmethod @@ -138,10 +137,9 @@ class Assistant(Role): async def main(): - options = Config().runtime_options - cost_manager = CostManager(**options) + cost_manager = CostManager() topic = "what's apple" - role = Assistant(options=options, cost_manager=cost_manager, language="Chinese") + role = Assistant(cost_manager=cost_manager, language="Chinese") await role.talk(topic) while True: has_action = await role.think() @@ -156,4 +154,5 @@ async def main(): if __name__ == '__main__': + CONFIG.language = "Chinese" asyncio.run(main()) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 1fd36e78c..e9bb55bed 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -13,7 +13,6 @@ import base64 import sys sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger from aiofile import async_open from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer @@ -109,8 +108,6 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti if __name__ == "__main__": - initialize_environment() - loop = asyncio.new_event_loop() v = loop.create_task(oas3_azsure_tts("测试，test")) loop.run_until_complete(v) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 624bb7d93..5c23f6566 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -13,13 +13,10 @@ import sys import connexion sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment def oas_http_svc(): """Start the OAS 3.0 OpenAPI HTTP service""" - initialize_environment() - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index bc551134a..43d22961b 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -17,7 +17,6 @@ import requests from pydantic import BaseModel sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -104,8 +103,6 @@ async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url if __name__ == "__main__": - initialize_environment() - v = oas3_metagpt_text_to_image("Panda emoji") data = base64.b64decode(v) with open("tmp.png", mode="wb") as writer: diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index cd48c62af..052a429ae 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -17,7 +17,6 @@ import requests from pydantic import BaseModel sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.utils.common import initialize_environment from metagpt.logs import logger @@ -96,7 +95,5 @@ async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_a if __name__ == "__main__": - initialize_environment() - v = oas3_openai_text_to_image("Panda emoji") print(v) diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 41d429109..0a2ca4071 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -14,12 +14,9 @@ from pathlib import Path sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.const import WORKSPACE_ROOT from metagpt.tools.azure_tts import AzureTTS -from metagpt.utils.common import initialize_environment def test_azure_tts(): - initialize_environment() - azure_tts = AzureTTS(subscription_key="", region="") text = """ 女儿看见父亲走了进来，问道： From 58369c4e3a402d9cb04142579fbc0ad9421f9559 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 21:01:15 +0800 Subject: [PATCH 127/378] feat: fix coding --- metagpt/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/config.py b/metagpt/config.py index f1c869b6c..05949408d 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -50,7 +50,7 @@ class Config(metaclass=Singleton): if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): - raise NotConfiguredException("Set OPENAI_API_KEY or Anthropic_API_KEY first") + logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") if not self.openai_api_base or "YOUR_API_BASE" == self.openai_api_base: openai_proxy = self._get("OPENAI_PROXY") or self.global_proxy From e201bf71d912542a4b4541528881583cb28e128a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:04:06 +0800 Subject: [PATCH 128/378] fixbug: CONFIG initialization --- metagpt/config.py | 17 +++++-- metagpt/provider/openai_api.py | 88 ++++------------------------------ metagpt/roles/role.py | 7 +-- metagpt/software_company.py | 7 +-- metagpt/utils/cost_manager.py | 79 ++++++++++++++++++++++++++++++ 5 files changed, 109 insertions(+), 89 deletions(-) create mode 100644 metagpt/utils/cost_manager.py diff --git a/metagpt/config.py b/metagpt/config.py index 05949408d..4cae79b17 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -4,6 +4,7 @@ Provide configuration, singleton. @Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ +import json import os from copy import deepcopy from typing import Any @@ -14,6 +15,7 @@ import yaml from metagpt.const import PROJECT_ROOT, OPTIONS from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType +from metagpt.utils.cost_manager import CostManager from metagpt.utils.singleton import Singleton @@ -43,12 +45,17 @@ class Config(metaclass=Singleton): def __init__(self, yaml_file=default_yaml_file): self._init_with_config_files_and_env(yaml_file) + self.cost_manager = CostManager(**json.loads(self.COST_MANAGER)) if self.COST_MANAGER else CostManager() + logger.info("Config loading done.") + self._update() + + def _update(self): self.global_proxy = self._get("GLOBAL_PROXY") self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -78,8 +85,7 @@ class Config(metaclass=Singleton): self.long_term_memory = self._get("LONG_TERM_MEMORY", False) if self.long_term_memory: logger.warning("LONG_TERM_MEMORY is True") - self.max_budget = self._get("MAX_BUDGET", 10.0) - self.total_cost = 0.0 + self.cost_manager.max_budget = self._get("MAX_BUDGET", 10.0) self.puppeteer_config = self._get("PUPPETEER_CONFIG", "") self.mmdc = self._get("MMDC", "mmdc") @@ -109,7 +115,8 @@ class Config(metaclass=Singleton): return m.get(*args, **kwargs) def get(self, key, *args, **kwargs): - """Retrieve values from config/key.yaml, config/config.yaml, and environment variables. Throw an error if not found.""" + """Retrieve values from config/key.yaml, config/config.yaml, and environment variables. + Throw an error if not found.""" value = self._get(key, *args, **kwargs) if value is None: raise ValueError(f"Key '{key}' not found in environment variables or in the YAML file") @@ -127,10 +134,12 @@ class Config(metaclass=Singleton): opts = deepcopy(OPTIONS.get()) opts.update(options) OPTIONS.set(opts) + self._update() @property def options(self): """Return all key-values""" return OPTIONS.get() + CONFIG = Config() diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7dba00530..e4dfade78 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -11,19 +11,18 @@ import re import time import random -from typing import NamedTuple, List +from typing import List import traceback import openai from openai.error import APIConnectionError -from pydantic import BaseModel from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI +from metagpt.utils.cost_manager import Costs from metagpt.utils.token_counter import ( - TOKEN_COSTS, count_message_tokens, count_string_tokens, get_max_completion_tokens, @@ -55,73 +54,6 @@ class RateLimiter: self.last_call_time = time.time() -class Costs(NamedTuple): - total_prompt_tokens: int - total_completion_tokens: int - total_cost: float - total_budget: float - - -class CostManager(BaseModel): - """计算使用接口的开销""" - - total_prompt_tokens: int = 0 - total_completion_tokens: int = 0 - total_budget: float = 0 - max_budget: float = CONFIG.max_budget - total_cost: float = 0 - - def update_cost(self, prompt_tokens, completion_tokens, model): - """ - Update the total cost, prompt tokens, and completion tokens. - - Args: - prompt_tokens (int): The number of tokens used in the prompt. - completion_tokens (int): The number of tokens used in the completion. - model (str): The model used for the API call. - """ - self.total_prompt_tokens += prompt_tokens - self.total_completion_tokens += completion_tokens - cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ - "completion"]) / 1000 - self.total_cost += cost - logger.info( - f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " - f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" - ) - - def get_total_prompt_tokens(self): - """ - Get the total number of prompt tokens. - - Returns: - int: The total number of prompt tokens. - """ - return self.total_prompt_tokens - - def get_total_completion_tokens(self): - """ - Get the total number of completion tokens. - - Returns: - int: The total number of completion tokens. - """ - return self.total_completion_tokens - - def get_total_cost(self): - """ - Get the total cost of API calls. - - Returns: - float: The total cost of API calls. - """ - return self.total_cost - - def get_costs(self) -> Costs: - """获得所有开销""" - return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) - - def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") logger.warning(""" @@ -136,12 +68,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): Check https://platform.openai.com/examples for examples """ - def __init__(self, cost_manager=None): + def __init__(self): self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False - self._cost_manager = cost_manager or CostManager() RateLimiter.__init__(self, rpm=self.rpm) def __init_openai(self, config): @@ -155,9 +86,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await self.async_retry_call(openai.ChatCompletion.acreate, - **self._cons_kwargs(messages), - stream=True - ) + **self._cons_kwargs(messages), + stream=True + ) # create variables to collect the stream of chunks collected_chunks = [] collected_messages = [] @@ -276,12 +207,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): try: prompt_tokens = int(usage['prompt_tokens']) completion_tokens = int(usage['completion_tokens']) - self._cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) + CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) except Exception as e: logger.error("updating costs failed!", e) def get_costs(self) -> Costs: - return self._cost_manager.get_costs() + return CONFIG.cost_manager.get_costs() def get_max_tokens(self, messages: list[dict]): if not self.auto_max_tokens: @@ -366,7 +297,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return None, input_string @staticmethod - async def async_retry_call(func, *args, **kwargs): + async def async_retry_call(func, *args, **kwargs): for i in range(OpenAIGPTAPI.MAX_TRY): try: rsp = await func(*args, **kwargs) @@ -399,4 +330,3 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index a1ac0d9e7..5d2cce802 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -9,13 +9,14 @@ """ from __future__ import annotations -from typing import Iterable, Type, Dict +from typing import Iterable, Type + from pydantic import BaseModel, Field -from metagpt.config import Config, CONFIG +from metagpt.config import CONFIG from metagpt.const import OPTIONS -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM, CostManager +from metagpt.llm import LLM from metagpt.actions import Action, ActionOutput from metagpt.logs import logger from metagpt.memory import Memory, LongTermMemory diff --git a/metagpt/software_company.py b/metagpt/software_company.py index 8d9c990ee..cfa3bd492 100644 --- a/metagpt/software_company.py +++ b/metagpt/software_company.py @@ -35,12 +35,13 @@ class SoftwareCompany(BaseModel): def invest(self, investment: float): """Invest company. raise NoMoneyException when exceed max_budget.""" self.investment = investment - CONFIG.max_budget = investment + CONFIG.cost_manager.max_budget = investment logger.info(f'Investment: ${investment}.') def _check_balance(self): - if CONFIG.total_cost > CONFIG.max_budget: - raise NoMoneyException(CONFIG.total_cost, f'Insufficient funds: {CONFIG.max_budget}') + if CONFIG.cost_manager.total_cost > CONFIG.cost_manager.max_budget: + raise NoMoneyException(CONFIG.cost_manager.total_cost, + f'Insufficient funds: {CONFIG.cost_manager.max_budget}') def start_project(self, idea, role="BOSS", cause_by=BossRequirement, **kwargs): """Start a project from publishing boss requirement.""" diff --git a/metagpt/utils/cost_manager.py b/metagpt/utils/cost_manager.py new file mode 100644 index 000000000..21b37d552 --- /dev/null +++ b/metagpt/utils/cost_manager.py @@ -0,0 +1,79 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/28 +@Author : mashenquan +@File : openai.py +@Desc : mashenquan, 2023/8/28. Separate the `CostManager` class to support user-level cost accounting. +""" + +from pydantic import BaseModel +from metagpt.logs import logger +from metagpt.utils.token_counter import TOKEN_COSTS +from typing import NamedTuple + + +class Costs(NamedTuple): + total_prompt_tokens: int + total_completion_tokens: int + total_cost: float + total_budget: float + + +class CostManager(BaseModel): + """Calculate the overhead of using the interface.""" + + total_prompt_tokens: int = 0 + total_completion_tokens: int = 0 + total_budget: float = 0 + max_budget: float = 10.0 + total_cost: float = 0 + + def update_cost(self, prompt_tokens, completion_tokens, model): + """ + Update the total cost, prompt tokens, and completion tokens. + + Args: + prompt_tokens (int): The number of tokens used in the prompt. + completion_tokens (int): The number of tokens used in the completion. + model (str): The model used for the API call. + """ + self.total_prompt_tokens += prompt_tokens + self.total_completion_tokens += completion_tokens + cost = (prompt_tokens * TOKEN_COSTS[model]["prompt"] + completion_tokens * TOKEN_COSTS[model][ + "completion"]) / 1000 + self.total_cost += cost + logger.info( + f"Total running cost: ${self.total_cost:.3f} | Max budget: ${self.max_budget:.3f} | " + f"Current cost: ${cost:.3f}, prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}" + ) + + def get_total_prompt_tokens(self): + """ + Get the total number of prompt tokens. + + Returns: + int: The total number of prompt tokens. + """ + return self.total_prompt_tokens + + def get_total_completion_tokens(self): + """ + Get the total number of completion tokens. + + Returns: + int: The total number of completion tokens. + """ + return self.total_completion_tokens + + def get_total_cost(self): + """ + Get the total cost of API calls. + + Returns: + float: The total cost of API calls. + """ + return self.total_cost + + def get_costs(self) -> Costs: + """获得所有开销""" + return Costs(self.total_prompt_tokens, self.total_completion_tokens, self.total_cost, self.total_budget) From 2a5b263371491a4be1799812d5dbd2f08c4c92c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:09:40 +0800 Subject: [PATCH 129/378] fixbug: CONFIG initialization --- metagpt/roles/assistant.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 944b250f1..57cb28e67 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -20,12 +20,10 @@ from pathlib import Path from metagpt.actions import ActionOutput from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction from metagpt.actions.talk_action import TalkAction -from metagpt.config import Config, CONFIG -from metagpt.const import BRAIN_MEMORY, SKILL_PATH +from metagpt.config import CONFIG from metagpt.learn.skill_loader import SkillLoader from metagpt.logs import logger from metagpt.memory.brain_memory import BrainMemory, MessageType -from metagpt.provider.openai_api import CostManager from metagpt.roles import Role from metagpt.schema import Message @@ -137,9 +135,8 @@ class Assistant(Role): async def main(): - cost_manager = CostManager() topic = "what's apple" - role = Assistant(cost_manager=cost_manager, language="Chinese") + role = Assistant(language="Chinese") await role.talk(topic) while True: has_action = await role.think() From b904607aab0e0c5567c785444e7a449852465bc3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:46:34 +0800 Subject: [PATCH 130/378] fixbug: async --- metagpt/actions/skill_action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index c921a5f17..e5bd32dae 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -76,16 +76,16 @@ class SkillAction(Action): async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" - self.rsp = self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) @staticmethod - def find_and_call_function(function_name, args, **kwargs): + async def find_and_call_function(function_name, args, **kwargs): try: module = importlib.import_module("metagpt.learn") function = getattr(module, function_name) # 调用函数并返回结果 - result = function(**args, **kwargs) + result = await function(**args, **kwargs) return result except (ModuleNotFoundError, AttributeError): logger.error(f"{function_name} not found") From 1903da126fe3802b5558e3366f0052c55e19298b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 28 Aug 2023 22:59:35 +0800 Subject: [PATCH 131/378] fixbug: async --- metagpt/actions/skill_action.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index e5bd32dae..fb801b454 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -9,6 +9,7 @@ import ast import importlib +import traceback from metagpt.actions import Action, ActionOutput from metagpt.learn.skill_loader import Skill @@ -76,7 +77,11 @@ class SkillAction(Action): async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" - self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + try: + self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + except Exception as e: + logger.exception(f"{e}, traceback:{traceback.format_exc()}") + self.rsp = f"Error: {e}" return ActionOutput(content=self.rsp, instruct_content=self._skill.json()) @staticmethod From 2ba457a6096afaa3b7d34d78fbaa17844aae552c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 10:24:06 +0800 Subject: [PATCH 132/378] feat: +exception catch --- metagpt/provider/openai_api.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e4dfade78..75ac38860 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -323,6 +323,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): except openai.error.RateLimitError as e: logger.warning(f"Exception:{e}") continue + except (openai.error.AuthenticationError, + openai.error.PermissionError, + openai.error.InvalidAPIType, + openai.error.SignatureVerificationError) as e: + logger.warning(f"Exception:{e}") + raise e except Exception as e: error_str = traceback.format_exc() logger.error(f"Exception:{e}, stack:{error_str}") From 91b7552f09a69cfc672480b1df3701c0b3c9a8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 11:33:50 +0800 Subject: [PATCH 133/378] fixbug: fix get_by_tags --- metagpt/roles/role.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 5d2cce802..aba7d4574 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -95,7 +95,9 @@ class RoleContext(BaseModel): @property def prerequisite(self): """Retrieve information with `prerequisite` tag""" - return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + if self.memory and hasattr(self.memory, 'get_by_tags'): + return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + return "" class Role: From 0aaf04100cd09d138dcf211d314fb8b22b85b36d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 11:40:13 +0800 Subject: [PATCH 134/378] fixbug: fix get_by_tags --- metagpt/roles/role.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index aba7d4574..efb8db9f8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -92,13 +92,6 @@ class RoleContext(BaseModel): def history(self) -> list[Message]: return self.memory.get() - @property - def prerequisite(self): - """Retrieve information with `prerequisite` tag""" - if self.memory and hasattr(self.memory, 'get_by_tags'): - return self.memory.get_by_tags([MessageTag.Prerequisite.value]) - return "" - class Role: """Role/Proxy""" @@ -208,7 +201,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - requirement = self._rc.important_memory or self._rc.prerequisite + requirement = self._rc.important_memory response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): From 14068cdc19613e78e94654ed898c77c310dce81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 14:35:35 +0800 Subject: [PATCH 135/378] fixbug: get user query empty --- metagpt/memory/memory.py | 8 ++++++++ metagpt/roles/role.py | 9 ++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index a96aaf1be..bf9f0541c 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -85,3 +85,11 @@ class Memory: continue rsp += self.index[action] return rsp + + def get_by_tags(self, tags: list) -> list[Message]: + """Return messages with specified tags""" + result = [] + for m in self.storage: + if m.is_contain_tags(tags): + result.append(m) + return result diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index efb8db9f8..aba7d4574 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -92,6 +92,13 @@ class RoleContext(BaseModel): def history(self) -> list[Message]: return self.memory.get() + @property + def prerequisite(self): + """Retrieve information with `prerequisite` tag""" + if self.memory and hasattr(self.memory, 'get_by_tags'): + return self.memory.get_by_tags([MessageTag.Prerequisite.value]) + return "" + class Role: """Role/Proxy""" @@ -201,7 +208,7 @@ class Role: # history=self.history) logger.info(f"{self._setting}: ready to {self._rc.todo}") - requirement = self._rc.important_memory + requirement = self._rc.important_memory or self._rc.prerequisite response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): From 9da450f8a77297067dd7d20940e875b466387823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 16:32:37 +0800 Subject: [PATCH 136/378] feat: + safe code --- metagpt/config.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/config.py b/metagpt/config.py index 4cae79b17..5944fef57 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -131,6 +131,8 @@ class Config(metaclass=Singleton): def set_context(self, options: dict): """Update current config""" + if not options: + return opts = deepcopy(OPTIONS.get()) opts.update(options) OPTIONS.set(opts) From ef6ec8c8c75181608a0e8be52278a9311e334770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 29 Aug 2023 20:52:45 +0800 Subject: [PATCH 137/378] fixbug: annotation --- examples/write_teaching_plan.py | 7 +++++-- metagpt/actions/action.py | 1 + metagpt/actions/skill_action.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/write_teaching_plan.py b/examples/write_teaching_plan.py index 191547193..c3a647b94 100644 --- a/examples/write_teaching_plan.py +++ b/examples/write_teaching_plan.py @@ -5,15 +5,18 @@ @Author : mashenquan @File : write_teaching_plan.py @Desc: Write teaching plan demo + ``` + export PYTHONPATH=$PYTHONPATH:$PWD + python examples/write_teaching_plan.py --language=Chinese --teaching_language=English + + ``` """ import asyncio from pathlib import Path -import sys from metagpt.config import CONFIG -sys.path.append(str(Path(__file__).resolve().parent.parent)) import aiofiles import fire from metagpt.logs import logger diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5cf4f3d81..c38c4e1b0 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,6 +6,7 @@ @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. """ +from __future__ import annotations from abc import ABC from typing import Optional diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index fb801b454..3ef0087fc 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -6,7 +6,7 @@ @File : skill_action.py @Desc : Call learned skill """ - +from __future__ import annotations import ast import importlib import traceback From dc14770e3d5ad327ec90e61c52346b9549d567fb Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 30 Aug 2023 10:53:47 +0800 Subject: [PATCH 138/378] separate workspace --- metagpt/actions/action.py | 12 +- metagpt/actions/design_api.py | 62 ++++------ metagpt/actions/project_management.py | 25 ++-- metagpt/actions/write_code.py | 24 +--- metagpt/actions/write_prd.py | 34 +++++- metagpt/config.py | 11 +- metagpt/roles/engineer.py | 94 +++++++-------- metagpt/roles/qa_engineer.py | 8 +- metagpt/roles/role.py | 43 +++---- metagpt/roles/teacher.py | 44 ++++--- metagpt/tools/sd_engine.py | 3 +- metagpt/utils/mermaid.py | 164 +++++++++++++------------- tests/metagpt/roles/ui_role.py | 4 +- tests/metagpt/tools/test_azure_tts.py | 17 +-- tests/metagpt/tools/test_sd_tool.py | 5 +- 15 files changed, 275 insertions(+), 275 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index 5cf4f3d81..e4b9613ad 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -6,6 +6,8 @@ @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. """ +from __future__ import annotations + from abc import ABC from typing import Optional @@ -13,12 +15,12 @@ from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput from metagpt.llm import LLM -from metagpt.utils.common import OutputParser from metagpt.logs import logger +from metagpt.utils.common import OutputParser class Action(ABC): - def __init__(self, name: str = '', context=None, llm: LLM = None): + def __init__(self, name: str = "", context=None, llm: LLM = None): self.name: str = name if llm is None: llm = LLM() @@ -49,9 +51,9 @@ class Action(ABC): return await self.llm.aask(prompt, system_msgs) @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) - async def _aask_v1(self, prompt: str, output_class_name: str, - output_data_mapping: dict, - system_msgs: Optional[list[str]] = None) -> ActionOutput: + async def _aask_v1( + self, prompt: str, output_class_name: str, output_data_mapping: dict, system_msgs: Optional[list[str]] = None + ) -> ActionOutput: """Append default prefix""" if not system_msgs: system_msgs = [] diff --git a/metagpt/actions/design_api.py b/metagpt/actions/design_api.py index cf23e6ad1..1c31b75fb 100644 --- a/metagpt/actions/design_api.py +++ b/metagpt/actions/design_api.py @@ -6,12 +6,12 @@ @File : design_api.py @Modified By: mashenquan, 2023-8-9, align `run` parameters with the parent :class:`Action` class. """ -import shutil -from pathlib import Path from typing import List -from metagpt.actions import Action, ActionOutput -from metagpt.const import WORKSPACE_ROOT +import aiofiles + +from metagpt.actions import Action +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.common import CodeParser from metagpt.utils.mermaid import mermaid_to_file @@ -93,52 +93,32 @@ OUTPUT_MAPPING = { class WriteDesign(Action): def __init__(self, name, context=None, llm=None): super().__init__(name, context, llm) - self.desc = "Based on the PRD, think about the system design, and design the corresponding APIs, " \ - "data structures, library tables, processes, and paths. Please provide your design, feedback " \ - "clearly and in detail." + self.desc = ( + "Based on the PRD, think about the system design, and design the corresponding APIs, " + "data structures, library tables, processes, and paths. Please provide your design, feedback " + "clearly and in detail." + ) - def recreate_workspace(self, workspace: Path): - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # 文件夹不存在，但我们不在意 - workspace.mkdir(parents=True, exist_ok=True) - - def _save_prd(self, docs_path, resources_path, prd): - prd_file = docs_path / 'prd.md' - quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) - mermaid_to_file(quadrant_chart, resources_path / 'competitive_analysis') - logger.info(f"Saving PRD to {prd_file}") - prd_file.write_text(prd) - - def _save_system_design(self, docs_path, resources_path, content): + async def _save_system_design(self, docs_path, resources_path, content): data_api_design = CodeParser.parse_code(block="Data structures and interface definitions", text=content) seq_flow = CodeParser.parse_code(block="Program call flow", text=content) - mermaid_to_file(data_api_design, resources_path / 'data_api_design') - mermaid_to_file(seq_flow, resources_path / 'seq_flow') - system_design_file = docs_path / 'system_design.md' + await mermaid_to_file(data_api_design, resources_path / "data_api_design") + await mermaid_to_file(seq_flow, resources_path / "seq_flow") + system_design_file = docs_path / "system_design.md" logger.info(f"Saving System Designs to {system_design_file}") - system_design_file.write_text(content) + async with aiofiles.open(system_design_file, "w") as f: + await f.write(content) - def _save(self, context, system_design): - if isinstance(system_design, ActionOutput): - content = system_design.content - ws_name = CodeParser.parse_str(block="Python package name", text=content) - else: - content = system_design - ws_name = CodeParser.parse_str(block="Python package name", text=system_design) - workspace = WORKSPACE_ROOT / ws_name - self.recreate_workspace(workspace) - docs_path = workspace / 'docs' - resources_path = workspace / 'resources' + async def _save(self, system_design: str): + workspace = CONFIG.workspace + docs_path = workspace / "docs" + resources_path = workspace / "resources" docs_path.mkdir(parents=True, exist_ok=True) resources_path.mkdir(parents=True, exist_ok=True) - self._save_prd(docs_path, resources_path, context[-1].content) - self._save_system_design(docs_path, resources_path, content) + await self._save_system_design(docs_path, resources_path, system_design) async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) - # system_design = await self._aask(prompt) system_design = await self._aask_v1(prompt, "system_design", OUTPUT_MAPPING) - self._save(context, system_design) + await self._save(system_design.content) return system_design diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 16473ff01..55e7cbcb5 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -8,11 +8,12 @@ """ from typing import List, Tuple -from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT -from metagpt.utils.common import CodeParser +import aiofiles -PROMPT_TEMPLATE = ''' +from metagpt.actions.action import Action +from metagpt.config import CONFIG + +PROMPT_TEMPLATE = """ # Context {context} @@ -37,7 +38,7 @@ Attention: Use '##' to split sections, not '#', and '## ' SHOULD W ## Anything UNCLEAR: Provide as Plain text. Make clear here. For example, don't forget a main entry. don't forget to init 3rd party libs. -''' +""" FORMAT_EXAMPLE = ''' --- @@ -103,23 +104,23 @@ OUTPUT_MAPPING = { class WriteTasks(Action): - def __init__(self, name="CreateTasks", context=None, llm=None): super().__init__(name, context, llm) - def _save(self, context, rsp): - ws_name = CodeParser.parse_str(block="Python package name", text=context[-1].content) - file_path = WORKSPACE_ROOT / ws_name / 'docs/api_spec_and_tasks.md' + async def _save(self, rsp): + file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" file_path.write_text(rsp.content) # Write requirements.txt - requirements_path = WORKSPACE_ROOT / ws_name / 'requirements.txt' - requirements_path.write_text(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) + requirements_path = CONFIG.workspace / "requirements.txt" + + async with aiofiles.open(requirements_path, "w") as f: + await f.write(rsp.instruct_content.dict().get("Required Python third-party packages").strip('"\n')) async def run(self, context, **kwargs): prompt = PROMPT_TEMPLATE.format(context=context, format_example=FORMAT_EXAMPLE) rsp = await self._aask_v1(prompt, "task", OUTPUT_MAPPING) - self._save(context, rsp) + await self._save(rsp) return rsp diff --git a/metagpt/actions/write_code.py b/metagpt/actions/write_code.py index cc122ef7a..fd54ce699 100644 --- a/metagpt/actions/write_code.py +++ b/metagpt/actions/write_code.py @@ -5,13 +5,12 @@ @Author : alexanderwu @File : write_code.py """ -from metagpt.actions import WriteDesign +from tenacity import retry, stop_after_attempt, wait_fixed + from metagpt.actions.action import Action -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger from metagpt.schema import Message from metagpt.utils.common import CodeParser -from tenacity import retry, stop_after_attempt, wait_fixed PROMPT_TEMPLATE = """ NOTICE @@ -49,23 +48,6 @@ class WriteCode(Action): def _is_invalid(self, filename): return any(i in filename for i in ["mp3", "wav"]) - def _save(self, context, filename, code): - # logger.info(filename) - # logger.info(code_rsp) - if self._is_invalid(filename): - return - - design = [i for i in context if i.cause_by == WriteDesign][0] - - ws_name = CodeParser.parse_str(block="Python package name", text=design.content) - ws_path = WORKSPACE_ROOT / ws_name - if f"{ws_name}/" not in filename and all(i not in filename for i in ["requirements.txt", ".md"]): - ws_path = ws_path / ws_name - code_path = ws_path / filename - code_path.parent.mkdir(parents=True, exist_ok=True) - code_path.write_text(code) - logger.info(f"Saving Code to {code_path}") - @retry(stop=stop_after_attempt(2), wait=wait_fixed(1)) async def write_code(self, prompt): code_rsp = await self._aask(prompt) @@ -74,7 +56,7 @@ class WriteCode(Action): async def run(self, context, filename): prompt = PROMPT_TEMPLATE.format(context=context, filename=filename) - logger.info(f'Writing {filename}..') + logger.info(f"Writing {filename}..") code = await self.write_code(prompt) # code_rsp = await self._aask_v1(prompt, "code_rsp", OUTPUT_MAPPING) # self._save(context, filename, code) diff --git a/metagpt/actions/write_prd.py b/metagpt/actions/write_prd.py index 0edd24d55..97f9138fd 100644 --- a/metagpt/actions/write_prd.py +++ b/metagpt/actions/write_prd.py @@ -7,9 +7,14 @@ """ from typing import List, Tuple +import aiofiles + from metagpt.actions import Action, ActionOutput from metagpt.actions.search_and_summarize import SearchAndSummarize +from metagpt.config import CONFIG from metagpt.logs import logger +from metagpt.utils.common import CodeParser +from metagpt.utils.mermaid import mermaid_to_file PROMPT_TEMPLATE = """ # Context @@ -121,7 +126,7 @@ OUTPUT_MAPPING = { "Competitive Quadrant Chart": (str, ...), "Requirement Analysis": (str, ...), "Requirement Pool": (List[Tuple[str, str]], ...), - "UI Design draft":(str, ...), + "UI Design draft": (str, ...), "Anything UNCLEAR": (str, ...), } @@ -139,8 +144,31 @@ class WritePRD(Action): logger.info(sas.result) logger.info(rsp) - prompt = PROMPT_TEMPLATE.format(requirements=requirements, search_information=info, - format_example=FORMAT_EXAMPLE) + prompt = PROMPT_TEMPLATE.format( + requirements=requirements, search_information=info, format_example=FORMAT_EXAMPLE + ) logger.debug(prompt) prd = await self._aask_v1(prompt, "prd", OUTPUT_MAPPING) + + await self._save(prd.content) return prd + + async def _save_prd(self, docs_path, resources_path, prd): + prd_file = docs_path / "prd.md" + quadrant_chart = CodeParser.parse_code(block="Competitive Quadrant Chart", text=prd) + await mermaid_to_file( + mermaid_code=quadrant_chart, output_file_without_suffix=resources_path / "competitive_analysis" + ) + async with aiofiles.open(prd_file, "w") as f: + await f.write(prd) + logger.info(f"Saving PRD to {prd_file}") + + async def _save(self, prd): + workspace = CONFIG.workspace + workspace.mkdir(parents=True, exist_ok=True) + + docs_path = workspace / "docs" + resources_path = workspace / "resources" + docs_path.mkdir(parents=True, exist_ok=True) + resources_path.mkdir(parents=True, exist_ok=True) + await self._save_prd(docs_path, resources_path, prd) diff --git a/metagpt/config.py b/metagpt/config.py index 5944fef57..908faaaaf 100644 --- a/metagpt/config.py +++ b/metagpt/config.py @@ -4,15 +4,17 @@ Provide configuration, singleton. @Modified BY: mashenquan, 2023/8/28. Replace the global variable `CONFIG` with `ContextVar`. """ +import datetime import json import os from copy import deepcopy from typing import Any +from uuid import uuid4 import openai import yaml -from metagpt.const import PROJECT_ROOT, OPTIONS +from metagpt.const import OPTIONS, PROJECT_ROOT, WORKSPACE_ROOT from metagpt.logs import logger from metagpt.tools import SearchEngineType, WebBrowserEngineType from metagpt.utils.cost_manager import CostManager @@ -55,7 +57,7 @@ class Config(metaclass=Singleton): self.openai_api_key = self._get("OPENAI_API_KEY") self.anthropic_api_key = self._get("Anthropic_API_KEY") if (not self.openai_api_key or "YOUR_API_KEY" == self.openai_api_key) and ( - not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key + not self.anthropic_api_key or "YOUR_API_KEY" == self.anthropic_api_key ): logger.warning("Set OPENAI_API_KEY or Anthropic_API_KEY first") self.openai_api_base = self._get("OPENAI_API_BASE") @@ -93,6 +95,11 @@ class Config(metaclass=Singleton): self.model_for_researcher_summary = self._get("MODEL_FOR_RESEARCHER_SUMMARY") self.model_for_researcher_report = self._get("MODEL_FOR_RESEARCHER_REPORT") + workspace_uid = ( + self._get("WORKSPACE_UID") or f"{datetime.datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[-8:]}" + ) + self.workspace = WORKSPACE_ROOT / workspace_uid + def _init_with_config_files_and_env(self, yaml_file): """从config/key.yaml / config/config.yaml / env三处按优先级递减加载""" configs = dict(os.environ) diff --git a/metagpt/roles/engineer.py b/metagpt/roles/engineer.py index 072e53998..97d0af087 100644 --- a/metagpt/roles/engineer.py +++ b/metagpt/roles/engineer.py @@ -6,17 +6,18 @@ @File : engineer.py """ import asyncio -import shutil from collections import OrderedDict from pathlib import Path -from metagpt.const import WORKSPACE_ROOT +import aiofiles + +from metagpt.actions import WriteCode, WriteCodeReview, WriteDesign, WriteTasks +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role -from metagpt.actions import WriteCode, WriteCodeReview, WriteTasks, WriteDesign from metagpt.schema import Message from metagpt.utils.common import CodeParser -from metagpt.utils.special_tokens import MSG_SEP, FILENAME_CODE_SEP +from metagpt.utils.special_tokens import FILENAME_CODE_SEP, MSG_SEP async def gather_ordered_k(coros, k) -> list: @@ -47,9 +48,15 @@ async def gather_ordered_k(coros, k) -> list: class Engineer(Role): - def __init__(self, name="Alex", profile="Engineer", goal="Write elegant, readable, extensible, efficient code", - constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", - n_borg=1, use_code_review=False): + def __init__( + self, + name="Alex", + profile="Engineer", + goal="Write elegant, readable, extensible, efficient code", + constraints="The code you write should conform to code standard like PEP8, be modular, easy to read and maintain", + n_borg=1, + use_code_review=False, + ): super().__init__(name, profile, goal, constraints) self._init_actions([WriteCode]) self.use_code_review = use_code_review @@ -72,31 +79,24 @@ class Engineer(Role): @classmethod def parse_workspace(cls, system_design_msg: Message) -> str: if system_design_msg.instruct_content: - return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip("\"") + return system_design_msg.instruct_content.dict().get("Python package name").strip().strip("'").strip('"') return CodeParser.parse_str(block="Python package name", text=system_design_msg.content) def get_workspace(self) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / 'src' + return CONFIG.workspace / "src" workspace = self.parse_workspace(msg) # Codes are written in workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace / workspace - def recreate_workspace(self): + async def write_file(self, filename: str, code: str): workspace = self.get_workspace() - try: - shutil.rmtree(workspace) - except FileNotFoundError: - pass # 文件夹不存在，但我们不在意 - workspace.mkdir(parents=True, exist_ok=True) - - def write_file(self, filename: str, code: str): - workspace = self.get_workspace() - filename = filename.replace('"', '').replace('\n', '') + filename = filename.replace('"', "").replace("\n", "") file = workspace / filename file.parent.mkdir(parents=True, exist_ok=True) - file.write_text(code) + async with aiofiles.open(file, "w") as f: + await f.write(code) return file def recv(self, message: Message) -> None: @@ -109,8 +109,7 @@ class Engineer(Role): todo_coros = [] for todo in self.todos: todo_coro = WriteCode().run( - context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), - filename=todo + context=self._rc.memory.get_by_actions([WriteTasks, WriteDesign]), filename=todo ) todo_coros.append(todo_coro) @@ -124,38 +123,40 @@ class Engineer(Role): self._rc.memory.add(msg) del self.todos[0] - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message(content="all done.", role=self.profile, cause_by=type(self._rc.todo)) return msg async def _act_sp(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + instruct_content = {} for todo in self.todos: - code = await WriteCode().run( - context=self._rc.history, - filename=todo - ) + code = await WriteCode().run(context=self._rc.history, filename=todo) # logger.info(todo) # logger.info(code_rsp) # code = self.parse_code(code_rsp) - file_path = self.write_file(todo, code) + file_path = await self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) + instruct_content[todo] = code - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + # code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = (todo, file_path) code_msg_all.append(code_msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), + instruct_content=instruct_content, role=self.profile, cause_by=type(self._rc.todo), - send_to="QaEngineer" + send_to="QaEngineer", ) return msg async def _act_sp_precision(self) -> Message: - code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + code_msg_all = [] # gather all code info, will pass to qa_engineer for tests later + instruct_content = {} for todo in self.todos: """ # 从历史信息中挑选必须的信息，以减少prompt长度（人工经验总结） @@ -170,35 +171,30 @@ class Engineer(Role): context.append(m.content) context_str = "\n".join(context) # 编写code - code = await WriteCode().run( - context=context_str, - filename=todo - ) + code = await WriteCode().run(context=context_str, filename=todo) # code review if self.use_code_review: try: - rewrite_code = await WriteCodeReview().run( - context=context_str, - code=code, - filename=todo - ) + rewrite_code = await WriteCodeReview().run(context=context_str, code=code, filename=todo) code = rewrite_code except Exception as e: logger.error("code review failed!", e) pass - file_path = self.write_file(todo, code) + file_path = await self.write_file(todo, code) msg = Message(content=code, role=self.profile, cause_by=WriteCode) self._rc.memory.add(msg) + instruct_content[todo] = code - code_msg = todo + FILENAME_CODE_SEP + str(file_path) + code_msg = (todo, file_path) code_msg_all.append(code_msg) - logger.info(f'Done {self.get_workspace()} generating.') + logger.info(f"Done {self.get_workspace()} generating.") msg = Message( - content=MSG_SEP.join(code_msg_all), + content=MSG_SEP.join(todo + FILENAME_CODE_SEP + str(file_path) for todo, file_path in code_msg_all), + instruct_content=instruct_content, role=self.profile, cause_by=type(self._rc.todo), - send_to="QaEngineer" + send_to="QaEngineer", ) return msg diff --git a/metagpt/roles/qa_engineer.py b/metagpt/roles/qa_engineer.py index 65bf2cc5b..491f5f997 100644 --- a/metagpt/roles/qa_engineer.py +++ b/metagpt/roles/qa_engineer.py @@ -9,7 +9,7 @@ import os from pathlib import Path from metagpt.actions import DebugError, RunCode, WriteCode, WriteDesign, WriteTest -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -43,13 +43,13 @@ class QaEngineer(Role): def get_workspace(self, return_proj_dir=True) -> Path: msg = self._rc.memory.get_by_action(WriteDesign)[-1] if not msg: - return WORKSPACE_ROOT / "src" + return CONFIG.workspace / "src" workspace = self.parse_workspace(msg) # project directory: workspace/{package_name}, which contains package source code folder, tests folder, resources folder, etc. if return_proj_dir: - return WORKSPACE_ROOT / workspace + return CONFIG.workspace / workspace # development codes directory: workspace/{package_name}/{package_name} - return WORKSPACE_ROOT / workspace / workspace + return CONFIG.workspace / workspace / workspace def write_file(self, filename: str, code: str): workspace = self.get_workspace() / "tests" diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index aba7d4574..2f0f713f8 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -11,15 +11,14 @@ from __future__ import annotations from typing import Iterable, Type - from pydantic import BaseModel, Field +from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import OPTIONS from metagpt.llm import LLM -from metagpt.actions import Action, ActionOutput from metagpt.logs import logger -from metagpt.memory import Memory, LongTermMemory +from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageTag PREFIX_TEMPLATE = """You are a {profile}, named {name}, your goal is {goal}, and the constraint is {constraints}. """ @@ -52,6 +51,7 @@ ROLE_TEMPLATE = """Your response should be based on the previous conversation hi class RoleSetting(BaseModel): """Role properties""" + name: str profile: str goal: str @@ -67,7 +67,8 @@ class RoleSetting(BaseModel): class RoleContext(BaseModel): """Runtime role context""" - env: 'Environment' = Field(default=None) + + env: "Environment" = Field(default=None) memory: Memory = Field(default_factory=Memory) long_term_memory: LongTermMemory = Field(default_factory=LongTermMemory) state: int = Field(default=0) @@ -95,7 +96,7 @@ class RoleContext(BaseModel): @property def prerequisite(self): """Retrieve information with `prerequisite` tag""" - if self.memory and hasattr(self.memory, 'get_by_tags'): + if self.memory and hasattr(self.memory, "get_by_tags"): return self.memory.get_by_tags([MessageTag.Prerequisite.value]) return "" @@ -145,7 +146,7 @@ class Role: logger.debug(self._actions) self._rc.todo = self._actions[self._rc.state] - def set_env(self, env: 'Environment'): + def set_env(self, env: "Environment"): """设置角色工作所处的环境，角色可以向环境说话，也可以通过观察接受环境消息""" self._rc.env = env @@ -192,12 +193,13 @@ class Role: self._set_state(0) return True prompt = self._get_prefix() - prompt += STATE_TEMPLATE.format(history=self._rc.history, states="\n".join(self._states), - n_states=len(self._states) - 1) + prompt += STATE_TEMPLATE.format( + history=self._rc.history, states="\n".join(self._states), n_states=len(self._states) - 1 + ) next_state = await self._llm.aask(prompt) logger.debug(f"{prompt=}") if not next_state.isdigit() or int(next_state) not in range(len(self._states)): - logger.warning(f'Invalid answer of state, {next_state=}') + logger.warning(f"Invalid answer of state, {next_state=}") next_state = "0" self._set_state(int(next_state)) return True @@ -212,8 +214,12 @@ class Role: response = await self._rc.todo.run(requirement) # logger.info(response) if isinstance(response, ActionOutput): - msg = Message(content=response.content, instruct_content=response.instruct_content, - role=self.profile, cause_by=type(self._rc.todo)) + msg = Message( + content=response.content, + instruct_content=response.instruct_content, + role=self.profile, + cause_by=type(self._rc.todo), + ) else: msg = Message(content=response, role=self.profile, cause_by=type(self._rc.todo)) self._rc.memory.add(msg) @@ -236,7 +242,7 @@ class Role: news_text = [f"{i.role}: {i.content[:20]}..." for i in self._rc.news] if news_text: - logger.debug(f'{self._setting} observed: {news_text}') + logger.debug(f"{self._setting} observed: {news_text}") return len(self._rc.news) def _publish_message(self, msg): @@ -310,20 +316,15 @@ class Role: def add_to_do(self, act): self._rc.todo = act - async def think(self) -> bool: + async def think(self) -> Action: """The exported `think` function""" - has_action = await self._think() - if not has_action: - return False - if not self._rc.todo: - return False - return True + await self._think() + return self._rc.todo async def act(self) -> ActionOutput: """The exported `act` function""" msg = await self._act() - return ActionOutput(content=msg.content, - instruct_content=msg.instruct_content) + return ActionOutput(content=msg.content, instruct_content=msg.instruct_content) @property def todo_description(self): diff --git a/metagpt/roles/teacher.py b/metagpt/roles/teacher.py index ca88fd681..031ce94c9 100644 --- a/metagpt/roles/teacher.py +++ b/metagpt/roles/teacher.py @@ -9,22 +9,34 @@ """ +import re + import aiofiles -from metagpt.actions.write_teaching_plan import WriteTeachingPlanPart, TeachingPlanRequirement -from metagpt.const import WORKSPACE_ROOT +from metagpt.actions.write_teaching_plan import ( + TeachingPlanRequirement, + WriteTeachingPlanPart, +) +from metagpt.config import CONFIG +from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message -from metagpt.logs import logger -import re class Teacher(Role): """Support configurable teacher roles, with native and teaching languages being replaceable through configurations.""" - def __init__(self, name='Lily', profile='{teaching_language} Teacher', - goal='writing a {language} teaching plan part by part', - constraints='writing in {language}', desc="", *args, **kwargs): + + def __init__( + self, + name="Lily", + profile="{teaching_language} Teacher", + goal="writing a {language} teaching plan part by part", + constraints="writing in {language}", + desc="", + *args, + **kwargs, + ): super().__init__(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs) actions = [] for topic in WriteTeachingPlanPart.TOPICS: @@ -54,7 +66,7 @@ class Teacher(Role): break logger.debug(f"{self._setting}: {self._rc.state=}, will do {self._rc.todo}") msg = await self._act() - if ret.content != '': + if ret.content != "": ret.content += "\n\n\n" ret.content += msg.content logger.info(ret.content) @@ -64,14 +76,14 @@ class Teacher(Role): async def save(self, content): """Save teaching plan""" filename = Teacher.new_file_name(self.course_title) - pathname = WORKSPACE_ROOT / "teaching_plan" + pathname = CONFIG.workspace / "teaching_plan" pathname.mkdir(exist_ok=True) pathname = pathname / filename try: - async with aiofiles.open(str(pathname), mode='w', encoding='utf-8') as writer: + async with aiofiles.open(str(pathname), mode="w", encoding="utf-8") as writer: await writer.write(content) except Exception as e: - logger.error(f'Save failed：{e}') + logger.error(f"Save failed：{e}") logger.info(f"Save to:{pathname}") @staticmethod @@ -80,8 +92,8 @@ class Teacher(Role): # Define the special characters that need to be replaced. illegal_chars = r'[#@$%!*&\\/:*?"<>|\n\t \']' # Replace the special characters with underscores. - filename = re.sub(illegal_chars, '_', lesson_title) + ext - return re.sub(r'_+', '_', filename) + filename = re.sub(illegal_chars, "_", lesson_title) + ext + return re.sub(r"_+", "_", filename) @property def course_title(self): @@ -93,9 +105,9 @@ class Teacher(Role): if act.rsp is None: return default_title title = act.rsp.lstrip("# \n") - if '\n' in title: - ix = title.index('\n') - title = title[0: ix] + if "\n" in title: + ix = title.index("\n") + title = title[0:ix] return title return default_title diff --git a/metagpt/tools/sd_engine.py b/metagpt/tools/sd_engine.py index a63dbe5ac..c33f67a51 100644 --- a/metagpt/tools/sd_engine.py +++ b/metagpt/tools/sd_engine.py @@ -14,7 +14,6 @@ from aiohttp import ClientSession from PIL import Image, PngImagePlugin from metagpt.config import Config -from metagpt.const import WORKSPACE_ROOT from metagpt.logs import logger config = Config() @@ -81,7 +80,7 @@ class SDEngine: return self.payload def _save(self, imgs, save_name=""): - save_dir = WORKSPACE_ROOT / "resources" / "SD_Output" + save_dir = CONFIG.get_workspace() / "resources" / "SD_Output" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) batch_decode_base64_to_image(imgs, save_dir, save_name=save_name) diff --git a/metagpt/utils/mermaid.py b/metagpt/utils/mermaid.py index 1245671fb..15fd08625 100644 --- a/metagpt/utils/mermaid.py +++ b/metagpt/utils/mermaid.py @@ -6,19 +6,20 @@ @File : mermaid.py @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -import subprocess +import asyncio from pathlib import Path -from metagpt.config import Config +# from metagpt.utils.common import check_cmd_exists +import aiofiles + +from metagpt.config import CONFIG, Config from metagpt.const import PROJECT_ROOT from metagpt.logs import logger -from metagpt.utils.common import check_cmd_exists -def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: +async def mermaid_to_file(mermaid_code, output_file_without_suffix, width=2048, height=2048) -> int: """suffix: png/svg/pdf - :param options: runtime context options, created by `Config` class object and changed in flow pipeline :param mermaid_code: mermaid code :param output_file_without_suffix: output filename :param width: @@ -27,92 +28,87 @@ def mermaid_to_file(options, mermaid_code, output_file_without_suffix, width=204 """ # Write the Mermaid code to a temporary file tmp = Path(f"{output_file_without_suffix}.mmd") - tmp.write_text(mermaid_code, encoding="utf-8") + async with aiofiles.open(tmp, "w", encoding="utf-8") as f: + await f.write(mermaid_code) + # tmp.write_text(mermaid_code, encoding="utf-8") - if check_cmd_exists("mmdc") != 0: - logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") - return -1 + # if check_cmd_exists("mmdc") != 0: + # logger.warning("RUN `npm install -g @mermaid-js/mermaid-cli` to install mmdc") + # return -1 - for suffix in ["pdf", "svg", "png"]: + # for suffix in ["pdf", "svg", "png"]: + for suffix in ["png"]: output_file = f"{output_file_without_suffix}.{suffix}" # Call the `mmdc` command to convert the Mermaid code to a PNG logger.info(f"Generating {output_file}..") + cmds = [CONFIG.mmdc, "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)] - if options.get("puppeteer_config"): - subprocess.run( - [ - options.get("mmdc"), - "-p", - options.get("puppeteer_config"), - "-i", - str(tmp), - "-o", - output_file, - "-w", - str(width), - "-H", - str(height), - ] - ) - else: - subprocess.run([options.get("mmdc"), "-i", str(tmp), "-o", output_file, "-w", str(width), "-H", str(height)]) - return 0 - - -MMC1 = """classDiagram - class Main { - -SearchEngine search_engine - +main() str - } - class SearchEngine { - -Index index - -Ranking ranking - -Summary summary - +search(query: str) str - } - class Index { - -KnowledgeBase knowledge_base - +create_index(data: dict) - +query_index(query: str) list - } - class Ranking { - +rank_results(results: list) list - } - class Summary { - +summarize_results(results: list) str - } - class KnowledgeBase { - +update(data: dict) - +fetch_data(query: str) dict - } - Main --> SearchEngine - SearchEngine --> Index - SearchEngine --> Ranking - SearchEngine --> Summary - Index --> KnowledgeBase""" - -MMC2 = """sequenceDiagram - participant M as Main - participant SE as SearchEngine - participant I as Index - participant R as Ranking - participant S as Summary - participant KB as KnowledgeBase - M->>SE: search(query) - SE->>I: query_index(query) - I->>KB: fetch_data(query) - KB-->>I: return data - I-->>SE: return results - SE->>R: rank_results(results) - R-->>SE: return ranked_results - SE->>S: summarize_results(ranked_results) - S-->>SE: return summary - SE-->>M: return summary""" + if CONFIG.puppeteer_config: + cmds.extend(["-p", CONFIG.puppeteer_config]) + process = await asyncio.create_subprocess_exec(*cmds) + await process.wait() + return process.returncode if __name__ == "__main__": + MMC1 = """classDiagram + class Main { + -SearchEngine search_engine + +main() str + } + class SearchEngine { + -Index index + -Ranking ranking + -Summary summary + +search(query: str) str + } + class Index { + -KnowledgeBase knowledge_base + +create_index(data: dict) + +query_index(query: str) list + } + class Ranking { + +rank_results(results: list) list + } + class Summary { + +summarize_results(results: list) str + } + class KnowledgeBase { + +update(data: dict) + +fetch_data(query: str) dict + } + Main --> SearchEngine + SearchEngine --> Index + SearchEngine --> Ranking + SearchEngine --> Summary + Index --> KnowledgeBase""" + + MMC2 = """sequenceDiagram + participant M as Main + participant SE as SearchEngine + participant I as Index + participant R as Ranking + participant S as Summary + participant KB as KnowledgeBase + M->>SE: search(query) + SE->>I: query_index(query) + I->>KB: fetch_data(query) + KB-->>I: return data + I-->>SE: return results + SE->>R: rank_results(results) + R-->>SE: return ranked_results + SE->>S: summarize_results(ranked_results) + S-->>SE: return summary + SE-->>M: return summary""" + conf = Config() - mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC1, - output_file_without_suffix=PROJECT_ROOT / "tmp/1.png") - mermaid_to_file(options=conf.runtime_options, mermaid_code=MMC2, - output_file_without_suffix=PROJECT_ROOT / "tmp/2.png") + asyncio.run( + mermaid_to_file( + options=conf.runtime_options, mermaid_code=MMC1, output_file_without_suffix=PROJECT_ROOT / "tmp/1.png" + ) + ) + asyncio.run( + mermaid_to_file( + options=conf.runtime_options, mermaid_code=MMC2, output_file_without_suffix=PROJECT_ROOT / "tmp/2.png" + ) + ) diff --git a/tests/metagpt/roles/ui_role.py b/tests/metagpt/roles/ui_role.py index a45a89cde..8e9660e36 100644 --- a/tests/metagpt/roles/ui_role.py +++ b/tests/metagpt/roles/ui_role.py @@ -8,7 +8,7 @@ from functools import wraps from importlib import import_module from metagpt.actions import Action, ActionOutput, WritePRD -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.roles import Role from metagpt.schema import Message @@ -214,7 +214,7 @@ class UIDesign(Action): logger.info("Finish icon design using StableDiffusion API") async def _save(self, css_content, html_content): - save_dir = WORKSPACE_ROOT / "resources" / "codes" + save_dir = CONFIG.workspace / "resources" / "codes" if not os.path.exists(save_dir): os.makedirs(save_dir, exist_ok=True) # Save CSS and HTML content to files diff --git a/tests/metagpt/tools/test_azure_tts.py b/tests/metagpt/tools/test_azure_tts.py index 0a2ca4071..b7f94a19c 100644 --- a/tests/metagpt/tools/test_azure_tts.py +++ b/tests/metagpt/tools/test_azure_tts.py @@ -8,11 +8,8 @@ @Modified By: mashenquan, 2023-8-17, move to `tools` folder. """ import asyncio -import sys -from pathlib import Path -sys.path.append(str(Path(__file__).resolve().parent.parent.parent.parent)) # fix-bug: No module named 'metagpt' -from metagpt.const import WORKSPACE_ROOT +from metagpt.config import CONFIG from metagpt.tools.azure_tts import AzureTTS @@ -28,15 +25,13 @@ def test_azure_tts(): “Writing a binary file in Python is similar to writing a regular text file, but you'll work with bytes instead of strings.” """ - path = WORKSPACE_ROOT / "tts" + path = CONFIG.workspace / "tts" path.mkdir(exist_ok=True, parents=True) filename = path / "girl.wav" loop = asyncio.new_event_loop() - v = loop.create_task(azure_tts.synthesize_speech( - lang="zh-CN", - voice="zh-CN-XiaomoNeural", - text=text, - output_file=str(filename))) + v = loop.create_task( + azure_tts.synthesize_speech(lang="zh-CN", voice="zh-CN-XiaomoNeural", text=text, output_file=str(filename)) + ) result = loop.run_until_complete(v) print(result) @@ -45,5 +40,5 @@ def test_azure_tts(): # TODO: 这里如果要检验，还要额外加上对应的asr，才能确保前后生成是接近一致的，但现在还没有 -if __name__ == '__main__': +if __name__ == "__main__": test_azure_tts() diff --git a/tests/metagpt/tools/test_sd_tool.py b/tests/metagpt/tools/test_sd_tool.py index 77e53c7dc..89c97f5e8 100644 --- a/tests/metagpt/tools/test_sd_tool.py +++ b/tests/metagpt/tools/test_sd_tool.py @@ -4,7 +4,8 @@ # import os -from metagpt.tools.sd_engine import SDEngine, WORKSPACE_ROOT +from metagpt.config import CONFIG +from metagpt.tools.sd_engine import SDEngine def test_sd_engine_init(): @@ -21,5 +22,5 @@ def test_sd_engine_generate_prompt(): async def test_sd_engine_run_t2i(): sd_engine = SDEngine() await sd_engine.run_t2i(prompts=["test"]) - img_path = WORKSPACE_ROOT / "resources" / "SD_Output" / "output_0.png" + img_path = CONFIG.workspace / "resources" / "SD_Output" / "output_0.png" assert os.path.exists(img_path) == True From 43dda1edafc25df7c99c76efa2b31486fd75e710 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Wed, 30 Aug 2023 11:55:54 +0800 Subject: [PATCH 139/378] fix options error --- metagpt/actions/project_management.py | 3 ++- .../tools/web_browser_engine_playwright.py | 20 ++++++++----------- 2 files changed, 10 insertions(+), 13 deletions(-) diff --git a/metagpt/actions/project_management.py b/metagpt/actions/project_management.py index 55e7cbcb5..1062f8984 100644 --- a/metagpt/actions/project_management.py +++ b/metagpt/actions/project_management.py @@ -109,7 +109,8 @@ class WriteTasks(Action): async def _save(self, rsp): file_path = CONFIG.workspace / "docs/api_spec_and_tasks.md" - file_path.write_text(rsp.content) + async with aiofiles.open(file_path, "w") as f: + await f.write(rsp.content) # Write requirements.txt requirements_path = CONFIG.workspace / "requirements.txt" diff --git a/metagpt/tools/web_browser_engine_playwright.py b/metagpt/tools/web_browser_engine_playwright.py index 199f8a0d1..8eecc4f40 100644 --- a/metagpt/tools/web_browser_engine_playwright.py +++ b/metagpt/tools/web_browser_engine_playwright.py @@ -8,11 +8,11 @@ from __future__ import annotations import asyncio import sys from pathlib import Path -from typing import Literal, Dict +from typing import Literal from playwright.async_api import async_playwright -from metagpt.config import Config +from metagpt.config import CONFIG from metagpt.logs import logger from metagpt.utils.parse_html import WebPage @@ -28,20 +28,18 @@ class PlaywrightWrapper: def __init__( self, - options: Dict, browser_type: Literal["chromium", "firefox", "webkit"] | None = None, launch_kwargs: dict | None = None, **kwargs, ) -> None: - self.options = options if browser_type is None: - browser_type = options.get("playwright_browser_type") + browser_type = CONFIG.playwright_browser_type self.browser_type = browser_type launch_kwargs = launch_kwargs or {} - if options.get("global_proxy") and "proxy" not in launch_kwargs: + if CONFIG.global_proxy and "proxy" not in launch_kwargs: args = launch_kwargs.get("args", []) if not any(str.startswith(i, "--proxy-server=") for i in args): - launch_kwargs["proxy"] = {"server": options.get("global_proxy")} + launch_kwargs["proxy"] = {"server": CONFIG.global_proxy} self.launch_kwargs = launch_kwargs context_kwargs = {} if "ignore_https_errors" in kwargs: @@ -81,8 +79,8 @@ class PlaywrightWrapper: executable_path = Path(browser_type.executable_path) if not executable_path.exists() and "executable_path" not in self.launch_kwargs: kwargs = {} - if self.options.get("global_proxy"): - kwargs["env"] = {"ALL_PROXY": self.options.get("global_proxy")} + if CONFIG.global_proxy: + kwargs["env"] = {"ALL_PROXY": CONFIG.global_proxy} await _install_browsers(self.browser_type, **kwargs) if self._has_run_precheck: @@ -150,8 +148,6 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, browser_type: str = "chromium", **kwargs): - return await PlaywrightWrapper(options=Config().runtime_options, - browser_type=browser_type, - **kwargs).run(url, *urls) + return await PlaywrightWrapper(browser_type=browser_type, **kwargs).run(url, *urls) fire.Fire(main) From bc9eb5ea933bdc0750d1dd56efa3a00d5b6a0b7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 11:57:18 +0800 Subject: [PATCH 140/378] feat: +.agent-store-config.yaml.example --- .agent-store-config.yaml.example | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 .agent-store-config.yaml.example diff --git a/.agent-store-config.yaml.example b/.agent-store-config.yaml.example new file mode 100644 index 000000000..037a44ed4 --- /dev/null +++ b/.agent-store-config.yaml.example @@ -0,0 +1,9 @@ +role: + name: Teacher # Referenced the `Teacher` in `metagpt/roles/teacher.py`. + module: metagpt.roles.teacher # Referenced `metagpt/roles/teacher.py`. + skills: # Refer to the skill `name` of the published skill in `.well-known/skills.yaml`. + - name: text_to_speech + description: Text-to-speech + - name: text_to_image + description: Create a drawing based on the text. + From b07b9919a07aa5426a2b077cb35b3763b5b8af22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 14:52:00 +0800 Subject: [PATCH 141/378] fixbug: os.environ --- metagpt/tools/azure_tts.py | 12 +++++++----- metagpt/tools/metagpt_text_to_image.py | 13 ++++++++++--- metagpt/tools/openai_text_to_embedding.py | 9 +++++---- metagpt/tools/openai_text_to_image.py | 12 +++++++++--- tests/conftest.py | 6 ++++++ 5 files changed, 37 insertions(+), 15 deletions(-) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index e9bb55bed..3100e2a3a 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -12,11 +12,12 @@ from uuid import uuid4 import base64 import sys +from metagpt.config import CONFIG, Config + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger from aiofile import async_open from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer -import os class AzureTTS: @@ -27,8 +28,8 @@ class AzureTTS: :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. """ - self.subscription_key = subscription_key if subscription_key else os.environ.get('AZURE_TTS_SUBSCRIPTION_KEY') - self.region = region if region else os.environ.get('AZURE_TTS_REGION') + self.subscription_key = subscription_key if subscription_key else CONFIG.AZURE_TTS_SUBSCRIPTION_KEY + self.region = region if region else CONFIG.AZURE_TTS_REGION # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles async def synthesize_speech(self, lang, voice, text, output_file): @@ -87,9 +88,9 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti if not style: style = "affectionate" if not subscription_key: - subscription_key = os.environ.get("AZURE_TTS_SUBSCRIPTION_KEY") + subscription_key = CONFIG.AZURE_TTS_SUBSCRIPTION_KEY if not region: - region = os.environ.get("AZURE_TTS_REGION") + region = CONFIG.AZURE_TTS_REGION xml_value = AzureTTS.role_style_text(role=role, style=style, text=text) tts = AzureTTS(subscription_key=subscription_key, region=region) @@ -108,6 +109,7 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti if __name__ == "__main__": + Config() loop = asyncio.new_event_loop() v = loop.create_task(oas3_azsure_tts("测试，test")) loop.run_until_complete(v) diff --git a/metagpt/tools/metagpt_text_to_image.py b/metagpt/tools/metagpt_text_to_image.py index 43d22961b..c5a0b872f 100644 --- a/metagpt/tools/metagpt_text_to_image.py +++ b/metagpt/tools/metagpt_text_to_image.py @@ -6,6 +6,7 @@ @File : metagpt_text_to_image.py @Desc : MetaGPT Text-to-Image OAS3 api, which provides text-to-image functionality. """ +import asyncio import base64 import os import sys @@ -16,6 +17,8 @@ import aiohttp import requests from pydantic import BaseModel +from metagpt.config import CONFIG, Config + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -25,7 +28,7 @@ class MetaGPTText2Image: """ :param model_url: Model reset api url """ - self.model_url = model_url if model_url else os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL') + self.model_url = model_url if model_url else CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL async def text_2_image(self, text, size_type="512x512"): """Text to image @@ -98,12 +101,16 @@ async def oas3_metagpt_text_to_image(text, size_type: str = "512x512", model_url if not text: return "" if not model_url: - model_url = os.environ.get('METAGPT_TEXT_TO_IMAGE_MODEL_URL') + model_url = CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL return await MetaGPTText2Image(model_url).text_2_image(text, size_type=size_type) if __name__ == "__main__": - v = oas3_metagpt_text_to_image("Panda emoji") + Config() + loop = asyncio.new_event_loop() + task = loop.create_task(oas3_metagpt_text_to_image("Panda emoji")) + v = loop.run_until_complete(task) + print(v) data = base64.b64decode(v) with open("tmp.png", mode="wb") as writer: writer.write(data) diff --git a/metagpt/tools/openai_text_to_embedding.py b/metagpt/tools/openai_text_to_embedding.py index 73984aff6..86b58d71f 100644 --- a/metagpt/tools/openai_text_to_embedding.py +++ b/metagpt/tools/openai_text_to_embedding.py @@ -17,7 +17,7 @@ import requests from pydantic import BaseModel import sys -from metagpt.config import CONFIG +from metagpt.config import CONFIG, Config sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -48,7 +48,7 @@ class OpenAIText2Embedding: """ :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ - self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY async def text_2_embedding(self, text, model="text-embedding-ada-002"): """Text to embedding @@ -89,7 +89,8 @@ async def oas3_openai_text_to_embedding(text, model="text-embedding-ada-002", op if __name__ == "__main__": + Config() loop = asyncio.new_event_loop() - v = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) - loop.run_until_complete(v) + task = loop.create_task(oas3_openai_text_to_embedding("Panda emoji")) + v = loop.run_until_complete(task) print(v) diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 052a429ae..395fa8133 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -6,6 +6,7 @@ @File : openai_text_to_image.py @Desc : OpenAI Text-to-Image OAS3 api, which provides text-to-image functionality. """ +import asyncio import base64 import os import sys @@ -16,6 +17,8 @@ import aiohttp import requests from pydantic import BaseModel +from metagpt.config import CONFIG, Config + sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -25,7 +28,7 @@ class OpenAIText2Image: """ :param openai_api_key: OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys` """ - self.openai_api_key = openai_api_key if openai_api_key else os.environ.get('OPENAI_API_KEY') + self.openai_api_key = openai_api_key if openai_api_key else CONFIG.OPENAI_API_KEY async def text_2_image(self, text, size_type="1024x1024"): """Text to image @@ -90,10 +93,13 @@ async def oas3_openai_text_to_image(text, size_type: str = "1024x1024", openai_a if not text: return "" if not openai_api_key: - openai_api_key = os.environ.get("OPENAI_API_KEY") + openai_api_key = CONFIG.OPENAI_API_KEY return await OpenAIText2Image(openai_api_key).text_2_image(text, size_type=size_type) if __name__ == "__main__": - v = oas3_openai_text_to_image("Panda emoji") + Config() + loop = asyncio.new_event_loop() + task = loop.create_task(oas3_openai_text_to_image("Panda emoji")) + v = loop.run_until_complete(task) print(v) diff --git a/tests/conftest.py b/tests/conftest.py index feecc7715..8f5069bbe 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,9 @@ from unittest.mock import Mock import pytest +import pytest_asyncio +from metagpt.config import Config from metagpt.logs import logger from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio @@ -68,3 +70,7 @@ def proxy(): server = asyncio.get_event_loop().run_until_complete(asyncio.start_server(handle_client, "127.0.0.1", 0)) return "http://{}:{}".format(*server.sockets[0].getsockname()) + +@pytest.fixture(scope="session", autouse=True) +def init_config(): + Config() From d2d8bda61598438f9aaa100ce1850d1fbd488c80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 14:56:50 +0800 Subject: [PATCH 142/378] feat: update azure-cognitiveservices-speech==1.31.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed3f755c9..25a480a68 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,5 +40,5 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.30.0 +azure-cognitiveservices-speech==1.31.0 aiofile \ No newline at end of file From 8aff30a350df8eeb544807bea9b8ddd7b1cd7e39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 15:36:52 +0800 Subject: [PATCH 143/378] refactor: replace aiofile with aiofiles --- metagpt/tools/azure_tts.py | 3 ++- requirements.txt | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 3100e2a3a..0dc16d516 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -11,6 +11,7 @@ from pathlib import Path from uuid import uuid4 import base64 import sys +import aiofiles from metagpt.config import CONFIG, Config @@ -97,7 +98,7 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti filename = Path(__file__).resolve().parent / (str(uuid4()).replace("-", "") + ".wav") try: await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) - async with async_open(filename, mode="rb") as reader: + async with aiofiles.open(filename, mode="rb") as reader: data = await reader.read() base64_string = base64.b64encode(data).decode('utf-8') filename.unlink() diff --git a/requirements.txt b/requirements.txt index 25a480a68..ca7fcbfda 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,5 +40,4 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.31.0 -aiofile \ No newline at end of file +azure-cognitiveservices-speech==1.31.0 \ No newline at end of file From a5ab5948c9f914edbb63408cc255a5ce4b229a4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 17:30:12 +0800 Subject: [PATCH 144/378] fixbug: remove aiofile --- metagpt/tools/azure_tts.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/metagpt/tools/azure_tts.py b/metagpt/tools/azure_tts.py index 0dc16d516..6864faf10 100644 --- a/metagpt/tools/azure_tts.py +++ b/metagpt/tools/azure_tts.py @@ -7,18 +7,15 @@ @Desc : azure TTS OAS3 api, which provides text-to-speech functionality """ import asyncio +import base64 from pathlib import Path from uuid import uuid4 -import base64 -import sys + import aiofiles +from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer from metagpt.config import CONFIG, Config - -sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger -from aiofile import async_open -from azure.cognitiveservices.speech import AudioConfig, SpeechConfig, SpeechSynthesizer class AzureTTS: @@ -34,18 +31,17 @@ class AzureTTS: # 参数参考：https://learn.microsoft.com/zh-cn/azure/cognitive-services/speech-service/language-support?tabs=tts#voice-styles-and-roles async def synthesize_speech(self, lang, voice, text, output_file): - speech_config = SpeechConfig( - subscription=self.subscription_key, region=self.region) + speech_config = SpeechConfig(subscription=self.subscription_key, region=self.region) speech_config.speech_synthesis_voice_name = voice audio_config = AudioConfig(filename=output_file) - synthesizer = SpeechSynthesizer( - speech_config=speech_config, - audio_config=audio_config) + synthesizer = SpeechSynthesizer(speech_config=speech_config, audio_config=audio_config) # More detail: https://learn.microsoft.com/en-us/azure/ai-services/speech-service/speech-synthesis-markup-voice - ssml_string = "" \ - f"{text}" + ssml_string = ( + "" + f"{text}" + ) return synthesizer.speak_ssml_async(ssml_string).get() @@ -100,7 +96,7 @@ async def oas3_azsure_tts(text, lang="", voice="", style="", role="", subscripti await tts.synthesize_speech(lang=lang, voice=voice, text=xml_value, output_file=str(filename)) async with aiofiles.open(filename, mode="rb") as reader: data = await reader.read() - base64_string = base64.b64encode(data).decode('utf-8') + base64_string = base64.b64encode(data).decode("utf-8") filename.unlink() except Exception as e: logger.error(f"text:{text}, error:{e}") From 9428c256caf1f16971216c7a3e4b66603bf8a825 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 17:55:13 +0800 Subject: [PATCH 145/378] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 33 +++++++++++++++++++ .../metagpt/provider/test_metagpt_llm_api.py | 17 ++++++++++ 2 files changed, 50 insertions(+) create mode 100644 metagpt/provider/metagpt_llm_api.py create mode 100644 tests/metagpt/provider/test_metagpt_llm_api.py diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py new file mode 100644 index 000000000..bfd003fff --- /dev/null +++ b/metagpt/provider/metagpt_llm_api.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/30 +@Author : mashenquan +@File : metagpt_llm_api.py +@Desc : MetaGPT LLM related APIs +""" + +import openai + +from metagpt.config import CONFIG +from metagpt.provider import OpenAIGPTAPI +from metagpt.provider.openai_api import RateLimiter + + +class MetaGPTLLMAPI(OpenAIGPTAPI): + """MetaGPT LLM api""" + + def __init__(self): + self.__init_openai(CONFIG) + self.llm = openai + self.model = CONFIG.METAGPT_API_MODEL + self.auto_max_tokens = False + RateLimiter.__init__(self, rpm=self.rpm) + + def __init_openai(self, config): + openai.api_key = CONFIG.METAGPT_API_KEY + if config.openai_api_base: + openai.api_base = CONFIG.METAGPT_API_BASE + if config.openai_api_type: + openai.api_type = CONFIG.METAGPT_API_TYPE + openai.api_version = CONFIG.METAGPT_API_VERSION + self.rpm = int(config.get("RPM", 10)) diff --git a/tests/metagpt/provider/test_metagpt_llm_api.py b/tests/metagpt/provider/test_metagpt_llm_api.py new file mode 100644 index 000000000..9c8356ca6 --- /dev/null +++ b/tests/metagpt/provider/test_metagpt_llm_api.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/30 +@Author : mashenquan +@File : test_metagpt_llm_api.py +""" +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI + + +def test_metagpt(): + llm = MetaGPTLLMAPI() + assert llm + + +if __name__ == "__main__": + test_metagpt() From 09fdb9d1ae1e5d0ab5f6a9c4571cff6bb265089f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:21:36 +0800 Subject: [PATCH 146/378] feat: +metagpt llm --- metagpt/const.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index 9e7462da6..e792ff35a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -14,9 +14,11 @@ def get_project_root(): """逐级向上寻找项目根目录""" current_path = Path.cwd() while True: - if (current_path / '.git').exists() or \ - (current_path / '.project_root').exists() or \ - (current_path / '.gitignore').exists(): + if ( + (current_path / ".git").exists() + or (current_path / ".project_root").exists() + or (current_path / ".gitignore").exists() + ): return current_path parent_path = current_path.parent if parent_path == current_path: @@ -25,15 +27,15 @@ def get_project_root(): PROJECT_ROOT = get_project_root() -DATA_PATH = PROJECT_ROOT / 'data' -WORKSPACE_ROOT = PROJECT_ROOT / 'workspace' -PROMPT_PATH = PROJECT_ROOT / 'metagpt/prompts' -UT_PATH = PROJECT_ROOT / 'data/ut' +DATA_PATH = PROJECT_ROOT / "data" +WORKSPACE_ROOT = PROJECT_ROOT / "workspace" +PROMPT_PATH = PROJECT_ROOT / "metagpt/prompts" +UT_PATH = PROJECT_ROOT / "data/ut" SWAGGER_PATH = UT_PATH / "files/api/" UT_PY_PATH = UT_PATH / "files/ut/" API_QUESTIONS_PATH = UT_PATH / "files/question/" YAPI_URL = "http://yapi.deepwisdomai.com/" -TMP = PROJECT_ROOT / 'tmp' +TMP = PROJECT_ROOT / "tmp" RESEARCH_PATH = DATA_PATH / "research" MEM_TTL = 24 * 30 * 3600 @@ -43,4 +45,12 @@ DEFAULT_LANGUAGE = "English" DEFAULT_MAX_TOKENS = 1500 COMMAND_TOKENS = 500 BRAIN_MEMORY = "BRAIN_MEMORY" -SKILL_PATH = "SKILL_PATH" \ No newline at end of file +SKILL_PATH = "SKILL_PATH" +SERPER_API_KEY = "SERPER_API_KEY" + +# MetaGPT LLM key defines +METAGPT_API_MODEL = "METAGPT_API_MODEL" +METAGPT_API_KEY = "METAGPT_API_KEY" +METAGPT_API_BASE = "METAGPT_API_BASE" +METAGPT_API_TYPE = "METAGPT_API_TYPE" +METAGPT_API_VERSION = "METAGPT_API_VERSION" From f65b959d5277053ddffebdc3fdc5e8a11af9c6b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:23:29 +0800 Subject: [PATCH 147/378] feat: +metagpt llm --- metagpt/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/const.py b/metagpt/const.py index e792ff35a..f2f1b4837 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -48,7 +48,7 @@ BRAIN_MEMORY = "BRAIN_MEMORY" SKILL_PATH = "SKILL_PATH" SERPER_API_KEY = "SERPER_API_KEY" -# MetaGPT LLM key defines +# Key Definitions for MetaGPT LLM METAGPT_API_MODEL = "METAGPT_API_MODEL" METAGPT_API_KEY = "METAGPT_API_KEY" METAGPT_API_BASE = "METAGPT_API_BASE" From 39e2e1d8a01be2696b3319f0b7c5794af7a650f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:25:10 +0800 Subject: [PATCH 148/378] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index bfd003fff..78a9e44b1 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -25,9 +25,9 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init_openai(self, config): openai.api_key = CONFIG.METAGPT_API_KEY - if config.openai_api_base: + if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE - if config.openai_api_type: + if CONFIG.METAGPT_API_TYPE: openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(config.get("RPM", 10)) From 4e92206301a43edfd6e777a1bff43e99acb884dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:26:52 +0800 Subject: [PATCH 149/378] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 78a9e44b1..bb8749e82 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -17,17 +17,17 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.__init_openai(CONFIG) + self.__init_openai() self.llm = openai self.model = CONFIG.METAGPT_API_MODEL self.auto_max_tokens = False RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): + def __init_openai(self): openai.api_key = CONFIG.METAGPT_API_KEY if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE if CONFIG.METAGPT_API_TYPE: openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION - self.rpm = int(config.get("RPM", 10)) + self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 From 01bdc2c90bcb8056f854c0560b6df7fa1137f43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 30 Aug 2023 19:28:13 +0800 Subject: [PATCH 150/378] feat: +metagpt llm --- metagpt/provider/metagpt_llm_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index bb8749e82..c27e7132d 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -23,7 +23,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): self.auto_max_tokens = False RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self): + def __init_openai(self, *args, **kwargs): openai.api_key = CONFIG.METAGPT_API_KEY if CONFIG.METAGPT_API_BASE: openai.api_base = CONFIG.METAGPT_API_BASE From d304e008a0d2d43ef538e22b821fb09568366272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 14:36:23 +0800 Subject: [PATCH 151/378] feat: +log --- metagpt/provider/base_gpt_api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f1590a77c..af0cf2ec0 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -15,7 +15,8 @@ from metagpt.provider.base_chatbot import BaseChatbot class BaseGPTAPI(BaseChatbot): """GPT API abstract class, requiring all inheritors to provide a series of standard capabilities""" - system_prompt = 'You are a helpful assistant.' + + system_prompt = "You are a helpful assistant." def _user_msg(self, msg: str) -> dict[str, str]: return {"role": "user", "content": msg} @@ -46,9 +47,9 @@ class BaseGPTAPI(BaseChatbot): rsp = await self.acompletion_text(message, stream=True) except Exception as e: logger.exception(f"{e}") + logger.info(f"ask:{msg}, error:{e}") raise e - logger.debug(message) - # logger.debug(rsp) + logger.info(f"ask:{msg}, anwser:{rsp}") return rsp def _extract_assistant_rsp(self, context): @@ -115,7 +116,7 @@ class BaseGPTAPI(BaseChatbot): def messages_to_prompt(self, messages: list[dict]): """[{"role": "user", "content": msg}] to user: etc.""" - return '\n'.join([f"{i['role']}: {i['content']}" for i in messages]) + return "\n".join([f"{i['role']}: {i['content']}" for i in messages]) def messages_to_dict(self, messages): """objects to [{"role": "user", "content": msg}] etc.""" From 8c2dfca68736eb74e749c609e480cd5d26ada18e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 21:03:21 +0800 Subject: [PATCH 152/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 64 +++++++++++++++++++---------- metagpt/tools/__init__.py | 5 +++ metagpt/tools/web_browser_engine.py | 28 ++++++------- 3 files changed, 61 insertions(+), 36 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 75ac38860..6d4e1b406 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -7,15 +7,21 @@ Change cost control from global to company level. """ import asyncio +import random import re import time -import random - -from typing import List import traceback +from typing import List + import openai from openai.error import APIConnectionError -from tenacity import retry, stop_after_attempt, after_log, wait_fixed, retry_if_exception_type +from tenacity import ( + after_log, + retry, + retry_if_exception_type, + stop_after_attempt, + wait_fixed, +) from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS @@ -40,7 +46,7 @@ class RateLimiter: self.rpm = rpm def split_batches(self, batch): - return [batch[i: i + self.rpm] for i in range(0, len(batch), self.rpm)] + return [batch[i : i + self.rpm] for i in range(0, len(batch), self.rpm)] async def wait_if_needed(self, num_requests): current_time = time.time() @@ -56,10 +62,12 @@ class RateLimiter: def log_and_reraise(retry_state): logger.error(f"Retry attempts exhausted. Last exception: {retry_state.outcome.exception()}") - logger.warning(""" + logger.warning( + """ Recommend going to https://deepwisdom.feishu.cn/wiki/MsGnwQBjiif9c3koSJNcYaoSnu4#part-XdatdVlhEojeAfxaaEZcMV3ZniQ See FAQ 5.8 -""") +""" + ) raise retry_state.outcome.exception() @@ -85,10 +93,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): self.rpm = int(config.get("RPM", 10)) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await self.async_retry_call(openai.ChatCompletion.acreate, - **self._cons_kwargs(messages), - stream=True - ) + response = await self.async_retry_call( + openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True + ) # create variables to collect the stream of chunks collected_chunks = [] collected_messages = [] @@ -151,7 +158,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @retry( stop=stop_after_attempt(3), wait=wait_fixed(1), - after=after_log(logger, logger.level('WARNING').name), + after=after_log(logger, logger.level("WARNING").name), retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) @@ -168,8 +175,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): try: prompt_tokens = count_message_tokens(messages, self.model) completion_tokens = count_string_tokens(rsp, self.model) - usage['prompt_tokens'] = prompt_tokens - usage['completion_tokens'] = completion_tokens + usage["prompt_tokens"] = prompt_tokens + usage["completion_tokens"] = completion_tokens return usage except Exception as e: logger.error("usage calculation failed!", e) @@ -205,8 +212,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): def _update_costs(self, usage: dict): if CONFIG.calc_usage: try: - prompt_tokens = int(usage['prompt_tokens']) - completion_tokens = int(usage['completion_tokens']) + prompt_tokens = int(usage["prompt_tokens"]) + completion_tokens = int(usage["completion_tokens"]) CONFIG.cost_manager.update_cost(prompt_tokens, completion_tokens, self.model) except Exception as e: logger.error("updating costs failed!", e) @@ -260,7 +267,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return result == "TRUE" async def rewrite(self, sentence: str, context: str): - command = f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + command = ( + f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + ) rsp = await self.aask(msg=command, system_msgs=[]) return rsp @@ -281,6 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): break w = text[idx:data_len] windows.append(w) + idx += data_len for i in range(len(windows)): if i + 1 == len(windows): break @@ -289,7 +299,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @staticmethod def extract_info(input_string): - pattern = r'\[([A-Z]+)\]:\s*(.+)' + pattern = r"\[([A-Z]+)\]:\s*(.+)" match = re.match(pattern, input_string) if match: return match.group(1), match.group(2) @@ -323,10 +333,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): except openai.error.RateLimitError as e: logger.warning(f"Exception:{e}") continue - except (openai.error.AuthenticationError, - openai.error.PermissionError, - openai.error.InvalidAPIType, - openai.error.SignatureVerificationError) as e: + except ( + openai.error.AuthenticationError, + openai.error.PermissionError, + openai.error.InvalidAPIType, + openai.error.SignatureVerificationError, + ) as e: logger.warning(f"Exception:{e}") raise e except Exception as e: @@ -336,3 +348,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + + +if __name__ == "__main__": + txt = """ +as dfas sad lkf sdkl sakdfsdk sjd jsk sdl sk dd sd asd fa sdf sad dd +- .gitlab-ci.yml & base_test.py + """ + OpenAIGPTAPI.split_texts(txt, 30) diff --git a/metagpt/tools/__init__.py b/metagpt/tools/__init__.py index d98087e4b..a148bb744 100644 --- a/metagpt/tools/__init__.py +++ b/metagpt/tools/__init__.py @@ -22,3 +22,8 @@ class WebBrowserEngineType(Enum): PLAYWRIGHT = "playwright" SELENIUM = "selenium" CUSTOM = "custom" + + @classmethod + def _missing_(cls, key): + """缺省类型转换""" + return cls.CUSTOM diff --git a/metagpt/tools/web_browser_engine.py b/metagpt/tools/web_browser_engine.py index da208dbc9..1f1a5ec67 100644 --- a/metagpt/tools/web_browser_engine.py +++ b/metagpt/tools/web_browser_engine.py @@ -6,29 +6,31 @@ from __future__ import annotations import importlib -from typing import Any, Callable, Coroutine, Literal, overload, Dict +from typing import Any, Callable, Coroutine, Dict, Literal, overload -from metagpt.config import Config +from metagpt.config import CONFIG from metagpt.tools import WebBrowserEngineType from metagpt.utils.parse_html import WebPage class WebBrowserEngine: def __init__( - self, - options: Dict, - engine: WebBrowserEngineType | None = None, - run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, + self, + options: Dict, + engine: WebBrowserEngineType | None = None, + run_func: Callable[..., Coroutine[Any, Any, WebPage | list[WebPage]]] | None = None, ): engine = engine or options.get("web_browser_engine") + if engine is None: + raise NotImplementedError - if engine == WebBrowserEngineType.PLAYWRIGHT: + if WebBrowserEngineType(engine) is WebBrowserEngineType.PLAYWRIGHT: module = "metagpt.tools.web_browser_engine_playwright" run_func = importlib.import_module(module).PlaywrightWrapper(options=options).run - elif engine == WebBrowserEngineType.SELENIUM: + elif WebBrowserEngineType(engine) is WebBrowserEngineType.SELENIUM: module = "metagpt.tools.web_browser_engine_selenium" run_func = importlib.import_module(module).SeleniumWrapper(options=options).run - elif engine == WebBrowserEngineType.CUSTOM: + elif WebBrowserEngineType(engine) is WebBrowserEngineType.CUSTOM: run_func = run_func else: raise NotImplementedError @@ -51,10 +53,8 @@ if __name__ == "__main__": import fire async def main(url: str, *urls: str, engine_type: Literal["playwright", "selenium"] = "playwright", **kwargs): - conf = Config() - return await WebBrowserEngine(options=conf.runtime_options, - engine=WebBrowserEngineType(engine_type), - **kwargs).run(url, *urls) - + return await WebBrowserEngine(options=CONFIG.options, engine=WebBrowserEngineType(engine_type), **kwargs).run( + url, *urls + ) fire.Fire(main) From 795b892b3530d7dc97248593be72c7561dfabbf9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:24:54 +0800 Subject: [PATCH 153/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 6d4e1b406..be262d606 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,6 +276,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): @staticmethod def split_texts(text: str, window_size) -> List[str]: """Splitting long text into sliding windows text""" + if window_size <= 0: + window_size = OpenAIGPTAPI.DEFAULT_TOKEN_SIZE total_len = len(text) if total_len <= window_size: return [text] @@ -348,6 +350,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 + DEFAULT_TOKEN_SIZE = 50 if __name__ == "__main__": From 67d08cb054cb863b1200a407b1d00bec42171c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:29:04 +0800 Subject: [PATCH 154/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index be262d606..dd5594b7d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -290,7 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if data_len + idx > total_len: windows.append(text[idx:]) break - w = text[idx:data_len] + w = text[idx : idx + data_len] windows.append(w) idx += data_len for i in range(len(windows)): From 614bdf9e742908be5e19a1fa938ec4fe135b2ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:43:58 +0800 Subject: [PATCH 155/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index dd5594b7d..64fbbdfd6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -286,13 +286,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): windows = [] idx = 0 while idx < total_len: - data_len = window_size - padding_size - if data_len + idx > total_len: + if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) break - w = text[idx : idx + data_len] + # 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # window_size=3, padding_size=1： + # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... + # idx=2, | idx=5 | idx=8 | ... + w = text[idx : idx + window_size] windows.append(w) - idx += data_len + idx += window_size - padding_size if idx == 0 else window_size + for i in range(len(windows)): if i + 1 == len(windows): break From 0156fa592248d613ca2d4110fe563d0275eedd28 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:48:50 +0800 Subject: [PATCH 156/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 64fbbdfd6..019ad0b8b 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -285,6 +285,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if window_size > 20 else 0 windows = [] idx = 0 + data_len = window_size - padding_size while idx < total_len: if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) @@ -295,7 +296,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # idx=2, | idx=5 | idx=8 | ... w = text[idx : idx + window_size] windows.append(w) - idx += window_size - padding_size if idx == 0 else window_size + idx += data_len for i in range(len(windows)): if i + 1 == len(windows): From ea35305b52040c3da7e9efbe1b1c104f3f7c0603 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 22:58:31 +0800 Subject: [PATCH 157/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 019ad0b8b..7ed9c0083 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -290,7 +290,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if window_size + idx > total_len: # 不足一个滑窗 windows.append(text[idx:]) break - # 第一个窗口少算自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] # window_size=3, padding_size=1： # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... # idx=2, | idx=5 | idx=8 | ... @@ -298,10 +298,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): windows.append(w) idx += data_len - for i in range(len(windows)): - if i + 1 == len(windows): - break - windows[i] += windows[i + 1][0:padding_size] return windows @staticmethod From 91595daa3b49f1a7bd0ed49e4bea80568455ba00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 31 Aug 2023 23:14:07 +0800 Subject: [PATCH 158/378] fixbug: dead loop --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7ed9c0083..14347f20c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -351,7 +351,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_TOKEN_SIZE = 50 + DEFAULT_TOKEN_SIZE = 500 if __name__ == "__main__": From 60d984f18478eeada59df09bde99e6bfae5fbe30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 10:25:31 +0800 Subject: [PATCH 159/378] fixbug: MET-1113 --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 555b202d1..e81f14bdd 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -14,7 +14,7 @@ from metagpt.logs import logger class TalkAction(Action): - def __init__(self, name: str = '', talk='', history_summary='', knowledge='', context=None, llm=None, **kwargs): + def __init__(self, name: str = "", talk="", history_summary="", knowledge="", context=None, llm=None, **kwargs): context = context or {} context["talk"] = talk context["history_summery"] = history_summary @@ -32,7 +32,7 @@ class TalkAction(Action): if self._history_summary != "": prompt += "According to the historical conversation above, " language = CONFIG.language or DEFAULT_LANGUAGE - prompt += f"Answer in {language}:\n {self._talk}" + prompt += f"Answer in {language}, and the answers must follow the Markdown format.\n {self._talk}" return prompt async def run(self, *args, **kwargs) -> ActionOutput: From 58dd5b8787a2df1523f4678815f48fc2e45ace55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 20:52:40 +0800 Subject: [PATCH 160/378] fixbug: exceed length --- metagpt/provider/openai_api.py | 18 +++++++++++------- metagpt/roles/assistant.py | 32 +++++++++++++++++++++----------- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 14347f20c..ac8feb738 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -242,14 +242,18 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """Generate text title""" max_response_token_count = 50 max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or DEFAULT_MAX_TOKENS - text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) + while True: + text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) - summaries = [] - for ws in text_windows: - response = await self.get_summary(ws) - summaries.append(response) - if len(summaries) == 1: - return summaries[0] + summaries = [] + for ws in text_windows: + response = await self.get_summary(ws) + summaries.append(response) + if len(summaries) == 1: + return summaries[0] + text = "\n".join(summaries) + if len(text) <= max_words * 2 and len(text) <= max_token_count: + break language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 57cb28e67..c681da65b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -18,7 +18,7 @@ import asyncio from pathlib import Path from metagpt.actions import ActionOutput -from metagpt.actions.skill_action import SkillAction, ArgumentsParingAction +from metagpt.actions.skill_action import ArgumentsParingAction, SkillAction from metagpt.actions.talk_action import TalkAction from metagpt.config import CONFIG from metagpt.learn.skill_loader import SkillLoader @@ -31,10 +31,19 @@ from metagpt.schema import Message class Assistant(Role): """Assistant for solving common issues.""" - def __init__(self, name="Lily", profile="An assistant", goal="Help to solve problem", - constraints="Talk in {language}", desc="", *args, **kwargs): - super(Assistant, self).__init__(name=name, profile=profile, - goal=goal, constraints=constraints, desc=desc, *args, **kwargs) + def __init__( + self, + name="Lily", + profile="An assistant", + goal="Help to solve problem", + constraints="Talk in {language}", + desc="", + *args, + **kwargs, + ): + super(Assistant, self).__init__( + name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs + ) brain_memory = CONFIG.BRAIN_MEMORY self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None @@ -65,8 +74,9 @@ class Assistant(Role): msg = Message(content=result) output = ActionOutput(content=result) else: - msg = Message(content=result.content, instruct_content=result.instruct_content, - cause_by=type(self._rc.todo)) + msg = Message( + content=result.content, instruct_content=result.instruct_content, cause_by=type(self._rc.todo) + ) output = result self.memory.add_answer(msg) return output @@ -85,8 +95,7 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, - **kwargs) + action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, **kwargs) self.add_to_do(action) return True @@ -111,7 +120,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_context_title(history_text, max_words=20) + history_summary = await self._llm.get_context_title(history_text, max_token_count_per_ask=1000, max_words=500) if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk @@ -122,6 +131,7 @@ class Assistant(Role): @staticmethod def extract_info(input_string): from metagpt.provider.openai_api import OpenAIGPTAPI + return OpenAIGPTAPI.extract_info(input_string) def get_memory(self) -> str: @@ -150,6 +160,6 @@ async def main(): await role.talk(talk) -if __name__ == '__main__': +if __name__ == "__main__": CONFIG.language = "Chinese" asyncio.run(main()) From ae414fccfadaf2d76faaf73f322c687e527c1b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:05:18 +0800 Subject: [PATCH 161/378] fixbug: exceed length --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index ac8feb738..c08a34f7e 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -247,7 +247,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): summaries = [] for ws in text_windows: - response = await self.get_summary(ws) + response = await self.get_summary(ws, max_words=max_response_token_count) summaries.append(response) if len(summaries) == 1: return summaries[0] From 3454761f950d49db49588eb35518708cf9d5b0e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:16:56 +0800 Subject: [PATCH 162/378] fixbug: exceed length --- metagpt/memory/brain_memory.py | 5 +++-- metagpt/roles/assistant.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index b3445a1f2..23b50afb3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -8,7 +8,7 @@ """ from enum import Enum -from typing import List, Dict +from typing import Dict, List import pydantic @@ -48,7 +48,7 @@ class BrainMemory(pydantic.BaseModel): texts = [Message(**m).content for m in self.history[:-1]] return "\n".join(texts) - def move_to_solution(self): + def move_to_solution(self, history_summary): if len(self.history) < 2: return msgs = self.history[:-1] @@ -58,6 +58,7 @@ class BrainMemory(pydantic.BaseModel): self.history = [] else: self.history = self.history[-1:] + self.history.insert(0, Message(content=history_summary)) @property def last_talk(self): diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c681da65b..719dfc29b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -125,7 +125,7 @@ class Assistant(Role): last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk - self.memory.move_to_solution() # Promptly clear memory after the issue is resolved. + self.memory.move_to_solution(history_summary) # Promptly clear memory after the issue is resolved. return last_talk @staticmethod From 7babb5ef711a2983fc9a726c77575fdf9d71014b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:18:11 +0800 Subject: [PATCH 163/378] fixbug: exceed length --- metagpt/memory/brain_memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 23b50afb3..9bafaafbb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -49,6 +49,7 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self, history_summary): + """放入solution队列，以备后续长程检索。目前还未加此功能""" if len(self.history) < 2: return msgs = self.history[:-1] From f2aaafbe001d094bdcbe059cad8a9378209f36ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:19:28 +0800 Subject: [PATCH 164/378] fixbug: exceed length --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 9bafaafbb..6bca9b140 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -59,7 +59,7 @@ class BrainMemory(pydantic.BaseModel): self.history = [] else: self.history = self.history[-1:] - self.history.insert(0, Message(content=history_summary)) + self.history.insert(0, Message(content="RESOLVED: " + history_summary)) @property def last_talk(self): From 8c943dd8e98f6e1dc60e6a534667900b2aa154bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:19:57 +0800 Subject: [PATCH 165/378] fixbug: exceed length --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 6bca9b140..c6be2cb7e 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -49,7 +49,7 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self, history_summary): - """放入solution队列，以备后续长程检索。目前还未加此功能""" + """放入solution队列，以备后续长程检索。目前还未加此功能，先用history_summary顶替""" if len(self.history) < 2: return msgs = self.history[:-1] From 478139c8dc2286d8e3db722145626a408cba4159 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Fri, 1 Sep 2023 21:21:47 +0800 Subject: [PATCH 166/378] feature: aioboto3 client --- config/config.yaml | 8 ++- metagpt/utils/s3.py | 127 +++++++++++++++++++++++++++++++++ requirements.txt | 4 +- tests/conftest.py | 7 +- tests/metagpt/utils/test_s3.py | 55 ++++++++++++++ 5 files changed, 198 insertions(+), 3 deletions(-) create mode 100644 metagpt/utils/s3.py create mode 100644 tests/metagpt/utils/test_s3.py diff --git a/config/config.yaml b/config/config.yaml index 88cca08e5..7c3d212f6 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -77,4 +77,10 @@ MODEL_FOR_RESEARCHER_SUMMARY: gpt-3.5-turbo MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k ### Meta Models -#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL \ No newline at end of file +#METAGPT_TEXT_TO_IMAGE_MODEL: MODEL_URL + +### S3 config +S3: + access_key: "YOUR_S3_ACCESS_KEY" + secret_key: "YOUR_S3_SECRET_KEY" + endpoint_url: "YOUR_S3_ENDPOINT_URL" \ No newline at end of file diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py new file mode 100644 index 000000000..2b4b8cb5f --- /dev/null +++ b/metagpt/utils/s3.py @@ -0,0 +1,127 @@ + +from typing import Optional + +import aioboto3 +from metagpt.logs import logger +from metagpt.config import Config + + +class S3: + """A class for interacting with Amazon S3 storage.""" + + def __init__(self): + self.session = aioboto3.Session() + self.s3_config = Config().get("S3") + self.auth_config = { + "service_name": "s3", + "aws_access_key_id": self.s3_config["access_key"], + "aws_secret_access_key": self.s3_config["secret_key"], + "endpoint_url": self.s3_config["endpoint_url"] + } + + async def upload_file( + self, + bucket: str, + local_path: str, + object_name: str, + ) -> None: + """Upload a file from the local path to the specified path of the storage bucket specified in s3. + + Args: + bucket: The name of the S3 storage bucket. + local_path: The local file path, including the file name. + object_name: The complete path of the uploaded file to be stored in S3, including the file name. + + Raises: + Exception: If an error occurs during the upload process, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + with open(local_path, "rb") as file: + await client.put_object(Body=file, Bucket=bucket, Key=object_name) + logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.") + except Exception as e: + logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}") + raise e + + async def get_object_url( + self, + bucket: str, + object_name: str, + ) -> str: + """Get the URL for a downloadable or preview file stored in the specified S3 bucket. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + + Returns: + The URL for the downloadable or preview file. + + Raises: + Exception: If an error occurs while retrieving the URL, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + file = await client.get_object(Bucket=bucket, Key=object_name) + return str(file["Body"].url) + except Exception as e: + logger.error(f"Failed to get the url for a downloadable or preview file: {e}") + raise e + + async def get_object( + self, + bucket: str, + object_name: str, + ) -> bytes: + """Get the binary data of a file stored in the specified S3 bucket. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + + Returns: + The binary data of the requested file. + + Raises: + Exception: If an error occurs while retrieving the file data, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + s3_object = await client.get_object(Bucket=bucket, Key=object_name) + return await s3_object["Body"].read() + except Exception as e: + logger.error(f"Failed to get the binary data of the file: {e}") + raise e + + async def download_file( + self, + bucket: str, + object_name: str, + local_path: str, + chunk_size: Optional[int] = 128 * 1024 + ) -> None: + """Download an S3 object to a local file. + + Args: + bucket: The name of the S3 storage bucket. + object_name: The complete path of the file stored in S3, including the file name. + local_path: The local file path where the S3 object will be downloaded. + chunk_size: The size of data chunks to read and write at a time. Default is 128 KB. + + Raises: + Exception: If an error occurs during the download process, an exception is raised. + """ + try: + async with self.session.client(**self.auth_config) as client: + s3_object = await client.get_object(Bucket=bucket, Key=object_name) + stream = s3_object["Body"] + with open(local_path, 'wb') as local_file: + while True: + file_data = await stream.read(chunk_size) + if not file_data: + break + local_file.write(file_data) + except Exception as e: + logger.error(f"Failed to download the file from S3: {e}") + raise e \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index ca7fcbfda..2e5112aba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -40,4 +40,6 @@ libcst==1.0.1 qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 -azure-cognitiveservices-speech==1.31.0 \ No newline at end of file +azure-cognitiveservices-speech==1.31.0 +aioboto3~=11.3.0 +pytest-asyncio~=0.21.1 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 8f5069bbe..0bc17bd6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -9,7 +9,6 @@ from unittest.mock import Mock import pytest -import pytest_asyncio from metagpt.config import Config from metagpt.logs import logger @@ -17,6 +16,8 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio import re +from metagpt.utils.s3 import S3 + class Context: def __init__(self): @@ -74,3 +75,7 @@ def proxy(): @pytest.fixture(scope="session", autouse=True) def init_config(): Config() + +@pytest.fixture(scope="session", autouse=True) +def s3(): + return S3() diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py new file mode 100644 index 000000000..760a976b0 --- /dev/null +++ b/tests/metagpt/utils/test_s3.py @@ -0,0 +1,55 @@ +import os +import pytest + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "local_path", "object_name"], + [ + ( + "agent-store", + "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", + "ui-designer/2023-09-01/1.png" + ) + ] +) +async def test_upload_file(s3, bucket, local_path, object_name): + await s3.upload_file(bucket=bucket, local_path=local_path, object_name=object_name) + s3_object = await s3.get_object(bucket=bucket, object_name=object_name) + assert s3_object + assert isinstance(s3_object, bytes) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "object_name"], + [("agent-store", "ui-designer/2023-09-01/1.png")] +) +async def test_get_object_url(s3, bucket, object_name): + url = await s3.get_object_url(bucket=bucket, object_name=object_name) + assert bucket in url + assert object_name in url + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "object_name"], + [("agent-store", "ui-designer/2023-09-01/1.png")] +) +async def test_get_object(s3, bucket, object_name): + s3_object = await s3.get_object(bucket=bucket, object_name=object_name) + assert s3_object + assert isinstance(s3_object, bytes) + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ["bucket", "local_path", "object_name"], + [ + ( + "agent-store", + "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", + "ui-designer/2023-09-01/1.png" + ) + ] +) +async def test_download_file(s3, bucket, local_path, object_name): + await s3.download_file(bucket=bucket, object_name=object_name, local_path=local_path) + assert os.path.exists(local_path) \ No newline at end of file From f7ebd2a3744b132fc606b3c4897eeb527dbb8aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:30:38 +0800 Subject: [PATCH 167/378] fixbug: exceed length --- metagpt/memory/brain_memory.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index c6be2cb7e..a5a3dbfc7 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -45,7 +45,16 @@ class BrainMemory(pydantic.BaseModel): def history_text(self): if len(self.history) == 0: return "" - texts = [Message(**m).content for m in self.history[:-1]] + texts = [] + for m in self.history[:-1]: + if isinstance(m, Dict): + t = Message(**m).content + elif isinstance(m, Message): + t = m.content + else: + continue + texts.append(t) + return "\n".join(texts) def move_to_solution(self, history_summary): From 760f7c5d5fce94638c70248053dc78b20afe47c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 21:32:27 +0800 Subject: [PATCH 168/378] fixbug: exceed length --- metagpt/roles/assistant.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 719dfc29b..fdd697b59 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -95,7 +95,10 @@ class Assistant(Role): return await handler(text, **kwargs) async def talk_handler(self, text, **kwargs) -> bool: - action = TalkAction(talk=text, knowledge=self.memory.get_knowledge(), llm=self._llm, **kwargs) + history = self.memory.history_text + action = TalkAction( + talk=text, knowledge=self.memory.get_knowledge(), history_summary=history, llm=self._llm, **kwargs + ) self.add_to_do(action) return True From 3e28b93e542f7223756cd127449b38001574a16a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 1 Sep 2023 22:46:04 +0800 Subject: [PATCH 169/378] refactor: refine prompt --- metagpt/actions/talk_action.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index e81f14bdd..ac395e9dd 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -32,7 +32,10 @@ class TalkAction(Action): if self._history_summary != "": prompt += "According to the historical conversation above, " language = CONFIG.language or DEFAULT_LANGUAGE - prompt += f"Answer in {language}, and the answers must follow the Markdown format.\n {self._talk}" + prompt += ( + f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " + f"{self._talk}" + ) return prompt async def run(self, *args, **kwargs) -> ActionOutput: From bfd8ed69e8676e204e60d94d25e52605d528f8b5 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Sat, 2 Sep 2023 10:55:38 +0800 Subject: [PATCH 170/378] update: delete pytest code --- requirements.txt | 3 +- tests/conftest.py | 5 ---- tests/metagpt/utils/test_s3.py | 55 ---------------------------------- 3 files changed, 1 insertion(+), 62 deletions(-) delete mode 100644 tests/metagpt/utils/test_s3.py diff --git a/requirements.txt b/requirements.txt index 2e5112aba..5daf710c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,5 +41,4 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 -aioboto3~=11.3.0 -pytest-asyncio~=0.21.1 \ No newline at end of file +aioboto3~=11.3.0 \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index 0bc17bd6a..98b45de7b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,6 @@ from metagpt.provider.openai_api import OpenAIGPTAPI as GPTAPI import asyncio import re -from metagpt.utils.s3 import S3 - class Context: def __init__(self): @@ -76,6 +74,3 @@ def proxy(): def init_config(): Config() -@pytest.fixture(scope="session", autouse=True) -def s3(): - return S3() diff --git a/tests/metagpt/utils/test_s3.py b/tests/metagpt/utils/test_s3.py deleted file mode 100644 index 760a976b0..000000000 --- a/tests/metagpt/utils/test_s3.py +++ /dev/null @@ -1,55 +0,0 @@ -import os -import pytest - - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "local_path", "object_name"], - [ - ( - "agent-store", - "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", - "ui-designer/2023-09-01/1.png" - ) - ] -) -async def test_upload_file(s3, bucket, local_path, object_name): - await s3.upload_file(bucket=bucket, local_path=local_path, object_name=object_name) - s3_object = await s3.get_object(bucket=bucket, object_name=object_name) - assert s3_object - assert isinstance(s3_object, bytes) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "object_name"], - [("agent-store", "ui-designer/2023-09-01/1.png")] -) -async def test_get_object_url(s3, bucket, object_name): - url = await s3.get_object_url(bucket=bucket, object_name=object_name) - assert bucket in url - assert object_name in url - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "object_name"], - [("agent-store", "ui-designer/2023-09-01/1.png")] -) -async def test_get_object(s3, bucket, object_name): - s3_object = await s3.get_object(bucket=bucket, object_name=object_name) - assert s3_object - assert isinstance(s3_object, bytes) - -@pytest.mark.asyncio -@pytest.mark.parametrize( - ["bucket", "local_path", "object_name"], - [ - ( - "agent-store", - "/code/send18-MetaGPT/workspace/resources/SD_Output/Flappy Bird_output_0.png", - "ui-designer/2023-09-01/1.png" - ) - ] -) -async def test_download_file(s3, bucket, local_path, object_name): - await s3.download_file(bucket=bucket, object_name=object_name, local_path=local_path) - assert os.path.exists(local_path) \ No newline at end of file From ca60cd0557effda735c4850b0f3b36fadd555fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 14:30:45 +0800 Subject: [PATCH 171/378] feat: +s3 --- metagpt/const.py | 3 ++ metagpt/learn/text_to_image.py | 22 +++++++++------ metagpt/learn/text_to_speech.py | 29 +++++++++++++------ metagpt/tools/openai_text_to_image.py | 38 +++++++++---------------- metagpt/utils/s3.py | 40 +++++++++++++++++++-------- 5 files changed, 79 insertions(+), 53 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index f2f1b4837..fbc2c928a 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -54,3 +54,6 @@ METAGPT_API_KEY = "METAGPT_API_KEY" METAGPT_API_BASE = "METAGPT_API_BASE" METAGPT_API_TYPE = "METAGPT_API_TYPE" METAGPT_API_VERSION = "METAGPT_API_VERSION" + +# format +BASE64_FORMAT = "base64" diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index 620e58180..c5f554ef3 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -6,10 +6,13 @@ @File : text_to_image.py @Desc : Text-to-Image skill, which provides text-to-image functionality. """ +import openai.error from metagpt.config import CONFIG +from metagpt.const import BASE64_FORMAT from metagpt.tools.metagpt_text_to_image import oas3_metagpt_text_to_image from metagpt.tools.openai_text_to_image import oas3_openai_text_to_image +from metagpt.utils.s3 import S3 async def text_to_image(text, size_type: str = "512x512", openai_api_key="", model_url="", **kwargs): @@ -23,13 +26,14 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod """ image_declaration = "data:image/png;base64," if CONFIG.METAGPT_TEXT_TO_IMAGE_MODEL_URL or model_url: - data = await oas3_metagpt_text_to_image(text, size_type, model_url) - return image_declaration + data if data else "" - - if CONFIG.OPENAI_API_KEY or openai_api_key: - data = await oas3_openai_text_to_image(text, size_type, openai_api_key) - return image_declaration + data if data else "" - - raise EnvironmentError - + base64_data = await oas3_metagpt_text_to_image(text, size_type, model_url) + elif CONFIG.OPENAI_API_KEY or openai_api_key: + base64_data = await oas3_openai_text_to_image(text, size_type, openai_api_key) + else: + raise openai.error.InvalidRequestError("缺少必要的参数") + s3 = S3() + url = await s3.cache(base64_data, BASE64_FORMAT) + if url: + return url + return image_declaration + base64_data if base64_data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 66fbba5be..7883ae9f3 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -6,14 +6,24 @@ @File : text_to_speech.py @Desc : Text-to-Speech skill, which provides text-to-speech functionality """ +import openai from metagpt.config import CONFIG - +from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.utils.s3 import S3 -async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style="affectionate", role="Girl", - subscription_key="", region="", **kwargs): +async def text_to_speech( + text, + lang="zh-CN", + voice="zh-CN-XiaomoNeural", + style="affectionate", + role="Girl", + subscription_key="", + region="", + **kwargs +): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -28,9 +38,12 @@ async def text_to_speech(text, lang="zh-CN", voice="zh-CN-XiaomoNeural", style=" """ audio_declaration = "data:audio/wav;base64," - if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or \ - (subscription_key and region): - data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) - return audio_declaration + data if data else data + if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): + base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) + s3 = S3() + url = await s3.cache(base64_data, BASE64_FORMAT) + if url: + return url + return audio_declaration + base64_data if base64_data else base64_data - raise EnvironmentError + raise openai.error.InvalidRequestError("缺少必要的参数") diff --git a/metagpt/tools/openai_text_to_image.py b/metagpt/tools/openai_text_to_image.py index 395fa8133..6025f04ba 100644 --- a/metagpt/tools/openai_text_to_image.py +++ b/metagpt/tools/openai_text_to_image.py @@ -8,18 +8,12 @@ """ import asyncio import base64 -import os -import sys -from pathlib import Path -from typing import List import aiohttp +import openai import requests -from pydantic import BaseModel from metagpt.config import CONFIG, Config - -sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: No module named 'metagpt' from metagpt.logs import logger @@ -37,27 +31,21 @@ class OpenAIText2Image: :param size_type: One of ['256x256', '512x512', '1024x1024'] :return: The image data is returned in Base64 encoding. """ - - class ImageUrl(BaseModel): - url: str - - class ImageResult(BaseModel): - data: List[ImageUrl] - created: int - - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {self.openai_api_key}" - } - data = {"prompt": text, "n": 1, "size": size_type} try: - async with aiohttp.ClientSession() as session: - async with session.post("https://api.openai.com/v1/images/generations", headers=headers, json=data) as response: - result = ImageResult(** await response.json()) - except requests.exceptions.RequestException as e: + result = await openai.Image.acreate( + api_key=CONFIG.OPENAI_API_KEY, + api_base=CONFIG.OPENAI_API_BASE, + api_type=None, + api_version=None, + organization=None, + prompt=text, + n=1, + size=size_type, + ) + except Exception as e: logger.error(f"An error occurred:{e}") return "" - if len(result.data) > 0: + if result and len(result.data) > 0: return await OpenAIText2Image.get_image_data(result.data[0].url) return "" diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 2b4b8cb5f..85837fedb 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -1,9 +1,14 @@ - +import base64 +import traceback +import uuid from typing import Optional import aioboto3 +import aiofiles + +from metagpt.config import CONFIG +from metagpt.const import BASE64_FORMAT, WORKSPACE_ROOT from metagpt.logs import logger -from metagpt.config import Config class S3: @@ -11,12 +16,12 @@ class S3: def __init__(self): self.session = aioboto3.Session() - self.s3_config = Config().get("S3") + self.s3_config = CONFIG.S3 self.auth_config = { "service_name": "s3", "aws_access_key_id": self.s3_config["access_key"], "aws_secret_access_key": self.s3_config["secret_key"], - "endpoint_url": self.s3_config["endpoint_url"] + "endpoint_url": self.s3_config["endpoint_url"], } async def upload_file( @@ -95,11 +100,7 @@ class S3: raise e async def download_file( - self, - bucket: str, - object_name: str, - local_path: str, - chunk_size: Optional[int] = 128 * 1024 + self, bucket: str, object_name: str, local_path: str, chunk_size: Optional[int] = 128 * 1024 ) -> None: """Download an S3 object to a local file. @@ -116,7 +117,7 @@ class S3: async with self.session.client(**self.auth_config) as client: s3_object = await client.get_object(Bucket=bucket, Key=object_name) stream = s3_object["Body"] - with open(local_path, 'wb') as local_file: + with open(local_path, "wb") as local_file: while True: file_data = await stream.read(chunk_size) if not file_data: @@ -124,4 +125,21 @@ class S3: local_file.write(file_data) except Exception as e: logger.error(f"Failed to download the file from S3: {e}") - raise e \ No newline at end of file + raise e + + async def cache(self, data: str, format: str = "") -> str: + """Save data to remote S3 and return url""" + object_name = str(uuid.uuid4()).replace("-", "") + pathname = WORKSPACE_ROOT / "s3_tmp" / object_name + try: + async with aiofiles.open(pathname, mode="w") as file: + if format == BASE64_FORMAT: + data = base64.b64decode(data) + await file.write(data) + + bucket = CONFIG.S3.get("bucket") + await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_name) + return await self.get_object_url(bucket=bucket, object_name=object_name) + except Exception as e: + logger.exception(f"{e}, stack:{traceback.format_exc()}") + return None From 578961ce2e07376e10c10191c80c9fc3714a22c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 15:24:48 +0800 Subject: [PATCH 172/378] feat: +role --- metagpt/roles/assistant.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index fdd697b59..48aff319b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,7 +54,14 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this sentence:\n {last_talk}\n" + prompt = "" + if CONFIG.agent_description: + prompt = ( + f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " + f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " + f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" + ) + prompt += f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" From a7b933311ebcaa18630947cebfbc96bda508231c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 15:39:17 +0800 Subject: [PATCH 173/378] feat: +role --- metagpt/actions/talk_action.py | 9 ++++++++- metagpt/roles/assistant.py | 9 +-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index ac395e9dd..4eed0d4f8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -27,7 +27,14 @@ class TalkAction(Action): @property def prompt(self): - prompt = f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt = "" + if CONFIG.agent_description: + prompt = ( + f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " + f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " + f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" + ) + prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" if self._history_summary != "": prompt += "According to the historical conversation above, " diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 48aff319b..fdd697b59 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,14 +54,7 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = "" - if CONFIG.agent_description: - prompt = ( - f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " - f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " - f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" - ) - prompt += f"Refer to this sentence:\n {last_talk}\n" + prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" From 07a1d229cf08f89595c10f7d198ca9aa6b0e550d Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 2 Sep 2023 18:03:31 +0800 Subject: [PATCH 174/378] restoresearch engine code --- metagpt/tools/search_engine.py | 33 ++++++++-------- metagpt/tools/search_engine_ddg.py | 48 +++++++++++------------ metagpt/tools/search_engine_googleapi.py | 13 +++--- metagpt/tools/search_engine_serpapi.py | 6 +-- metagpt/tools/search_engine_serper.py | 4 +- tests/metagpt/tools/test_search_engine.py | 19 +++++---- 6 files changed, 62 insertions(+), 61 deletions(-) diff --git a/metagpt/tools/search_engine.py b/metagpt/tools/search_engine.py index 5b8b7f046..db8c091d1 100644 --- a/metagpt/tools/search_engine.py +++ b/metagpt/tools/search_engine.py @@ -4,12 +4,11 @@ @Time : 2023/5/6 20:15 @Author : alexanderwu @File : search_engine.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import importlib -from typing import Callable, Coroutine, Literal, overload, Dict +from typing import Callable, Coroutine, Literal, overload from metagpt.config import CONFIG from metagpt.tools import SearchEngineType @@ -28,23 +27,23 @@ class SearchEngine: """ def __init__( - self, - engine: SearchEngineType | None = None, - run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None + self, + engine: SearchEngineType | None = None, + run_func: Callable[[str, int, bool], Coroutine[None, None, str | list[str]]] = None, ): engine = engine or CONFIG.search_engine if engine == SearchEngineType.SERPAPI_GOOGLE: module = "metagpt.tools.search_engine_serpapi" - run_func = importlib.import_module(module).SerpAPIWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).SerpAPIWrapper().run elif engine == SearchEngineType.SERPER_GOOGLE: module = "metagpt.tools.search_engine_serper" - run_func = importlib.import_module(module).SerperWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).SerperWrapper().run elif engine == SearchEngineType.DIRECT_GOOGLE: module = "metagpt.tools.search_engine_googleapi" - run_func = importlib.import_module(module).GoogleAPIWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).GoogleAPIWrapper().run elif engine == SearchEngineType.DUCK_DUCK_GO: module = "metagpt.tools.search_engine_ddg" - run_func = importlib.import_module(module).DDGAPIWrapper(**CONFIG.options).run + run_func = importlib.import_module(module).DDGAPIWrapper().run elif engine == SearchEngineType.CUSTOM_ENGINE: pass # run_func = run_func else: @@ -54,19 +53,19 @@ class SearchEngine: @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, ) -> list[dict[str, str]]: ... diff --git a/metagpt/tools/search_engine_ddg.py b/metagpt/tools/search_engine_ddg.py index 78562c77e..57bc61b82 100644 --- a/metagpt/tools/search_engine_ddg.py +++ b/metagpt/tools/search_engine_ddg.py @@ -1,14 +1,11 @@ #!/usr/bin/env python -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. -""" from __future__ import annotations import asyncio import json from concurrent import futures -from typing import Literal, overload, Optional +from typing import Literal, overload try: from duckduckgo_search import DDGS @@ -18,6 +15,8 @@ except ImportError: "You can install it by running the command: `pip install -e.[search-ddg]`" ) +from metagpt.config import CONFIG + class DDGAPIWrapper: """Wrapper around duckduckgo_search API. @@ -26,44 +25,43 @@ class DDGAPIWrapper: """ def __init__( - self, - *, - global_proxy: Optional[str] = None, - loop: asyncio.AbstractEventLoop | None = None, - executor: futures.Executor | None = None, + self, + *, + loop: asyncio.AbstractEventLoop | None = None, + executor: futures.Executor | None = None, ): kwargs = {} - if global_proxy: - kwargs["proxies"] = global_proxy + if CONFIG.global_proxy: + kwargs["proxies"] = CONFIG.global_proxy self.loop = loop self.executor = executor self.ddgs = DDGS(**kwargs) @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[True] = True, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[True] = True, + focus: list[str] | None = None, ) -> str: ... @overload def run( - self, - query: str, - max_results: int = 8, - as_string: Literal[False] = False, - focus: list[str] | None = None, + self, + query: str, + max_results: int = 8, + as_string: Literal[False] = False, + focus: list[str] | None = None, ) -> list[dict[str, str]]: ... async def run( - self, - query: str, - max_results: int = 8, - as_string: bool = True, + self, + query: str, + max_results: int = 8, + as_string: bool = True, ) -> str | list[dict]: """Return the results of a Google search using the official Google API diff --git a/metagpt/tools/search_engine_googleapi.py b/metagpt/tools/search_engine_googleapi.py index b5aeb5875..b9faf2ced 100644 --- a/metagpt/tools/search_engine_googleapi.py +++ b/metagpt/tools/search_engine_googleapi.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -""" -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. -""" from __future__ import annotations import asyncio @@ -14,6 +11,7 @@ from urllib.parse import urlparse import httplib2 from pydantic import BaseModel, validator +from metagpt.config import CONFIG from metagpt.logs import logger try: @@ -29,7 +27,6 @@ except ImportError: class GoogleAPIWrapper(BaseModel): google_api_key: Optional[str] = None google_cse_id: Optional[str] = None - global_proxy: Optional[str] = None loop: Optional[asyncio.AbstractEventLoop] = None executor: Optional[futures.Executor] = None @@ -39,6 +36,7 @@ class GoogleAPIWrapper(BaseModel): @validator("google_api_key", always=True) @classmethod def check_google_api_key(cls, val: str): + val = val or CONFIG.google_api_key if not val: raise ValueError( "To use, make sure you provide the google_api_key when constructing an object. Alternatively, " @@ -49,7 +47,8 @@ class GoogleAPIWrapper(BaseModel): @validator("google_cse_id", always=True) @classmethod - def check_google_cse_id(cls, val): + def check_google_cse_id(cls, val: str): + val = val or CONFIG.google_cse_id if not val: raise ValueError( "To use, make sure you provide the google_cse_id when constructing an object. Alternatively, " @@ -61,8 +60,8 @@ class GoogleAPIWrapper(BaseModel): @property def google_api_client(self): build_kwargs = {"developerKey": self.google_api_key} - if self.global_proxy: - parse_result = urlparse(self.global_proxy) + if CONFIG.global_proxy: + parse_result = urlparse(CONFIG.global_proxy) proxy_type = parse_result.scheme if proxy_type == "https": proxy_type = "http" diff --git a/metagpt/tools/search_engine_serpapi.py b/metagpt/tools/search_engine_serpapi.py index 1b93a91e9..750184198 100644 --- a/metagpt/tools/search_engine_serpapi.py +++ b/metagpt/tools/search_engine_serpapi.py @@ -4,14 +4,13 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator -from metagpt.config import Config +from metagpt.config import CONFIG class SerpAPIWrapper(BaseModel): @@ -33,6 +32,7 @@ class SerpAPIWrapper(BaseModel): @validator("serpapi_api_key", always=True) @classmethod def check_serpapi_api_key(cls, val: str): + val = val or CONFIG.serpapi_api_key if not val: raise ValueError( "To use, make sure you provide the serpapi_api_key when constructing an object. Alternatively, " @@ -112,4 +112,4 @@ class SerpAPIWrapper(BaseModel): if __name__ == "__main__": import fire - fire.Fire(SerpAPIWrapper(Config().runtime_options).run) + fire.Fire(SerpAPIWrapper().run) diff --git a/metagpt/tools/search_engine_serper.py b/metagpt/tools/search_engine_serper.py index 849839f05..0eec2694b 100644 --- a/metagpt/tools/search_engine_serper.py +++ b/metagpt/tools/search_engine_serper.py @@ -4,7 +4,6 @@ @Time : 2023/5/23 18:27 @Author : alexanderwu @File : search_engine_serpapi.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import json from typing import Any, Dict, Optional, Tuple @@ -12,6 +11,8 @@ from typing import Any, Dict, Optional, Tuple import aiohttp from pydantic import BaseModel, Field, validator +from metagpt.config import CONFIG + class SerperWrapper(BaseModel): search_engine: Any #: :meta private: @@ -25,6 +26,7 @@ class SerperWrapper(BaseModel): @validator("serper_api_key", always=True) @classmethod def check_serper_api_key(cls, val: str): + val = val or CONFIG.serper_api_key if not val: raise ValueError( "To use, make sure you provide the serper_api_key when constructing an object. Alternatively, " diff --git a/tests/metagpt/tools/test_search_engine.py b/tests/metagpt/tools/test_search_engine.py index 35ccdf78b..25bce124a 100644 --- a/tests/metagpt/tools/test_search_engine.py +++ b/tests/metagpt/tools/test_search_engine.py @@ -4,13 +4,11 @@ @Time : 2023/5/2 17:46 @Author : alexanderwu @File : test_search_engine.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ from __future__ import annotations import pytest -from metagpt.config import Config from metagpt.logs import logger from metagpt.tools import SearchEngineType from metagpt.tools.search_engine import SearchEngine @@ -18,7 +16,9 @@ from metagpt.tools.search_engine import SearchEngine class MockSearchEnine: async def run(self, query: str, max_results: int = 8, as_string: bool = True) -> str | list[dict[str, str]]: - rets = [{"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results)] + rets = [ + {"url": "https://metagpt.com/mock/{i}", "title": query, "snippet": query * i} for i in range(max_results) + ] return "\n".join(rets) if as_string else rets @@ -36,13 +36,16 @@ class MockSearchEnine: (SearchEngineType.DUCK_DUCK_GO, None, 6, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 8, False), (SearchEngineType.CUSTOM_ENGINE, MockSearchEnine().run, 6, False), - ], ) -async def test_search_engine(search_engine_typpe, run_func, max_results, as_string): - conf = Config() - search_engine = SearchEngine(options=conf.runtime_options, engine=search_engine_typpe, run_func=run_func) - rsp = await search_engine.run(query="metagpt", max_results=max_results, as_string=as_string) +async def test_search_engine( + search_engine_typpe, + run_func, + max_results, + as_string, +): + search_engine = SearchEngine(search_engine_typpe, run_func) + rsp = await search_engine.run("metagpt", max_results=max_results, as_string=as_string) logger.info(rsp) if as_string: assert isinstance(rsp, str) From c5e16330a21231abbf2f326889e941ce3a890995 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 18:51:46 +0800 Subject: [PATCH 175/378] feat: +path --- metagpt/utils/s3.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 85837fedb..d13030292 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -1,4 +1,5 @@ import base64 +import os.path import traceback import uuid from typing import Optional @@ -138,8 +139,11 @@ class S3: await file.write(data) bucket = CONFIG.S3.get("bucket") - await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_name) - return await self.get_object_url(bucket=bucket, object_name=object_name) + object_pathname = CONFIG.S3.get("path") or "system" + object_pathname += f"/{object_name}" + object_pathname = os.path.normpath(object_pathname) + await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_pathname) + return await self.get_object_url(bucket=bucket, object_name=object_pathname) except Exception as e: logger.exception(f"{e}, stack:{traceback.format_exc()}") return None From 2148e4e4f47edc8e108daf261fb1166b31012f8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 19:17:35 +0800 Subject: [PATCH 176/378] feat: +skill config --- metagpt/learn/skill_loader.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 1cd83240d..83200bca6 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -7,11 +7,13 @@ @Desc : Skill YAML Configuration Loader. """ from pathlib import Path -from typing import List, Dict, Optional +from typing import Dict, List, Optional import yaml from pydantic import BaseModel, Field +from metagpt.config import CONFIG + class Example(BaseModel): ask: str @@ -52,7 +54,7 @@ class SkillLoader: def __init__(self, skill_yaml_file_name: Path = None): if not skill_yaml_file_name: skill_yaml_file_name = Path(__file__).parent.parent.parent / ".well-known/skills.yaml" - with open(str(skill_yaml_file_name), 'r') as file: + with open(str(skill_yaml_file_name), "r") as file: skills = yaml.safe_load(file) self._skills = SkillsDeclaration(**skills) @@ -62,8 +64,18 @@ class SkillLoader: if not entity_skills: return {} + agent_skills = CONFIG.agent_skills + if not agent_skills: + return {} + + class AgentSkill(BaseModel): + name: str + + names = [AgentSkill(**i).name for i in agent_skills] description_to_name_mappings = {} for s in entity_skills.skills: + if s.name not in names: + continue description_to_name_mappings[s.description] = s.name return description_to_name_mappings From 610dd8b4ba2771bb7f1d38b101be7fb2cb425fa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 19:25:06 +0800 Subject: [PATCH 177/378] feat: +skill config --- metagpt/utils/s3.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index d13030292..531142737 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -131,9 +131,11 @@ class S3: async def cache(self, data: str, format: str = "") -> str: """Save data to remote S3 and return url""" object_name = str(uuid.uuid4()).replace("-", "") - pathname = WORKSPACE_ROOT / "s3_tmp" / object_name + path = WORKSPACE_ROOT / "s3_tmp" + path.mkdir(exist_ok=True) + pathname = path / object_name try: - async with aiofiles.open(pathname, mode="w") as file: + async with aiofiles.open(str(pathname), mode="w") as file: if format == BASE64_FORMAT: data = base64.b64decode(data) await file.write(data) @@ -142,7 +144,7 @@ class S3: object_pathname = CONFIG.S3.get("path") or "system" object_pathname += f"/{object_name}" object_pathname = os.path.normpath(object_pathname) - await self.upload_file(bucket=bucket, local_path=pathname, object_name=object_pathname) + await self.upload_file(bucket=bucket, local_path=str(pathname), object_name=object_pathname) return await self.get_object_url(bucket=bucket, object_name=object_pathname) except Exception as e: logger.exception(f"{e}, stack:{traceback.format_exc()}") From 86e3ca0ba99c7522cdbca9df35e3b8fc965fa384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 19:44:26 +0800 Subject: [PATCH 178/378] feat: +skill config --- metagpt/utils/s3.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 531142737..6df244197 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -2,13 +2,14 @@ import base64 import os.path import traceback import uuid +from pathlib import Path from typing import Optional import aioboto3 import aiofiles from metagpt.config import CONFIG -from metagpt.const import BASE64_FORMAT, WORKSPACE_ROOT +from metagpt.const import BASE64_FORMAT from metagpt.logs import logger @@ -131,8 +132,7 @@ class S3: async def cache(self, data: str, format: str = "") -> str: """Save data to remote S3 and return url""" object_name = str(uuid.uuid4()).replace("-", "") - path = WORKSPACE_ROOT / "s3_tmp" - path.mkdir(exist_ok=True) + path = Path(__file__).parent pathname = path / object_name try: async with aiofiles.open(str(pathname), mode="w") as file: @@ -145,7 +145,10 @@ class S3: object_pathname += f"/{object_name}" object_pathname = os.path.normpath(object_pathname) await self.upload_file(bucket=bucket, local_path=str(pathname), object_name=object_pathname) + pathname.unlink(missing_ok=True) + return await self.get_object_url(bucket=bucket, object_name=object_pathname) except Exception as e: logger.exception(f"{e}, stack:{traceback.format_exc()}") + pathname.unlink(missing_ok=True) return None From 7881937e8fb3c5a4ef183d6460fc1d741c0d6b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 20:47:14 +0800 Subject: [PATCH 179/378] feat: test s3 --- metagpt/utils/s3.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 6df244197..74c3f1654 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -129,13 +129,13 @@ class S3: logger.error(f"Failed to download the file from S3: {e}") raise e - async def cache(self, data: str, format: str = "") -> str: + async def cache(self, data: str, file_ext: str, format: str = "") -> str: """Save data to remote S3 and return url""" - object_name = str(uuid.uuid4()).replace("-", "") + object_name = str(uuid.uuid4()).replace("-", "") + file_ext path = Path(__file__).parent pathname = path / object_name try: - async with aiofiles.open(str(pathname), mode="w") as file: + async with aiofiles.open(str(pathname), mode="wb") as file: if format == BASE64_FORMAT: data = base64.b64decode(data) await file.write(data) From 9d74e8e157029ec1e49d307adc121772e1dc048f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 20:51:02 +0800 Subject: [PATCH 180/378] feat: test s3 --- metagpt/learn/text_to_image.py | 4 ++-- metagpt/learn/text_to_speech.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index c5f554ef3..dd85cf617 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -33,7 +33,7 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod raise openai.error.InvalidRequestError("缺少必要的参数") s3 = S3() - url = await s3.cache(base64_data, BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) if url: - return url + return f"[{text}]({url})" return image_declaration + base64_data if base64_data else "" diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 7883ae9f3..819da2364 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -22,7 +22,7 @@ async def text_to_speech( role="Girl", subscription_key="", region="", - **kwargs + **kwargs, ): """Text to speech For more details, check out:`https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts` @@ -41,9 +41,9 @@ async def text_to_speech( if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) s3 = S3() - url = await s3.cache(base64_data, BASE64_FORMAT) + url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) if url: - return url + return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data raise openai.error.InvalidRequestError("缺少必要的参数") From 7bd62b6a498543d4fdf95e62e643eebed8743c3f Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 2 Sep 2023 21:04:51 +0800 Subject: [PATCH 181/378] add google search skill --- .well-known/skills.yaml | 19 ++++++++++++++++ metagpt/learn/__init__.py | 6 ++--- metagpt/learn/google_search.py | 12 ++++++++++ tests/metagpt/learn/test_google_search.py | 27 +++++++++++++++++++++++ 4 files changed, 60 insertions(+), 4 deletions(-) create mode 100644 metagpt/learn/google_search.py create mode 100644 tests/metagpt/learn/test_google_search.py diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 06b9ffd0c..009368dbe 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -45,3 +45,22 @@ entities: returns: type: string format: base64 + + - name: web_search + description: Perform Google searches to provide real-time information. + id: web_search.web_search + x-prerequisite: + - name: SEARCH_ENGINE + description: "Supported values: serpapi/google/serper/ddg" + - name: SERPER_API_KEY + description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + arguments: + query: 'The search query. Required.' + max_results: 'The number of search results to retrieve. Default value: 6.' + examples: + - ask: 'Search for information about artificial intelligence' + answer: 'web_search(query="Search for information about artificial intelligence", max_results=6)' + - ask: 'Find news articles about climate change' + answer: 'web_search(query="Find news articles about climate change", max_results=6)' + returns: + type: string \ No newline at end of file diff --git a/metagpt/learn/__init__.py b/metagpt/learn/__init__.py index c8270dbfb..bab9f3e37 100644 --- a/metagpt/learn/__init__.py +++ b/metagpt/learn/__init__.py @@ -8,8 +8,6 @@ from metagpt.learn.text_to_image import text_to_image from metagpt.learn.text_to_speech import text_to_speech +from metagpt.learn.google_search import google_search -__all__ = [ - "text_to_image", - "text_to_speech", -] \ No newline at end of file +__all__ = ["text_to_image", "text_to_speech", "google_search"] diff --git a/metagpt/learn/google_search.py b/metagpt/learn/google_search.py new file mode 100644 index 000000000..ef099fe94 --- /dev/null +++ b/metagpt/learn/google_search.py @@ -0,0 +1,12 @@ +from metagpt.tools.search_engine import SearchEngine + + +async def google_search(query: str, max_results: int = 6, **kwargs): + """Perform a web search and retrieve search results. + + :param query: The search query. + :param max_results: The number of search results to retrieve + :return: The web search results in markdown format. + """ + resluts = await SearchEngine().run(query, max_results=max_results, as_string=False) + return "\n".join(f"{i}. [{j['title']}]({j['link']}): {j['snippet']}" for i, j in enumerate(resluts, 1)) diff --git a/tests/metagpt/learn/test_google_search.py b/tests/metagpt/learn/test_google_search.py new file mode 100644 index 000000000..da32e8923 --- /dev/null +++ b/tests/metagpt/learn/test_google_search.py @@ -0,0 +1,27 @@ +import asyncio + +from pydantic import BaseModel + +from metagpt.learn.google_search import google_search + + +async def mock_google_search(): + class Input(BaseModel): + input: str + + inputs = [{"input": "ai agent"}] + + for i in inputs: + seed = Input(**i) + result = await google_search(seed.input) + assert result != "" + + +def test_suite(): + loop = asyncio.get_event_loop() + task = loop.create_task(mock_google_search()) + loop.run_until_complete(task) + + +if __name__ == "__main__": + test_suite() From 842aac82fcda09a6879edfdcf40adfc12b053790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:11:44 +0800 Subject: [PATCH 182/378] fixbug: summary too long --- metagpt/provider/openai_api.py | 45 ++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index c08a34f7e..4764b6aad 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -226,38 +226,45 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=20): + async def get_summary(self, text: str, max_words=200): + max_token_count = DEFAULT_MAX_TOKENS + max_count = 100 + while max_count > 0: + if len(text) < max_token_count: + return await self._get_summary(text, max_words=max_words) + + text_windows = self.split_texts(text, window_size=max_token_count - max_words) + summaries = [] + for ws in text_windows: + response = await self._get_summary(ws, max_words=max_words) + summaries.append(response) + if len(summaries) == 1: + return summaries[0] + + # Merged and retry + text = "\n".join(summaries) + + max_count -= 1 # safeguard + raise openai.error.InvalidRequestError("text too long") + + async def _get_summary(self, text: str, max_words=20): """Generate text summary""" if len(text) < max_words: return text - language = CONFIG.language or DEFAULT_LANGUAGE - command = f"Translate the above content into a {language} summary of less than {max_words} words." + command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) logger.info(f"summary rsp: {response}") return response - async def get_context_title(self, text: str, max_token_count_per_ask=None, max_words=5) -> str: + async def get_context_title(self, text: str, max_words=5) -> str: """Generate text title""" - max_response_token_count = 50 - max_token_count = max_token_count_per_ask or CONFIG.MAX_TOKENS or DEFAULT_MAX_TOKENS - while True: - text_windows = self.split_texts(text, window_size=max_token_count - max_response_token_count) - - summaries = [] - for ws in text_windows: - response = await self.get_summary(ws, max_words=max_response_token_count) - summaries.append(response) - if len(summaries) == 1: - return summaries[0] - text = "\n".join(summaries) - if len(text) <= max_words * 2 and len(text) <= max_token_count: - break + summary = await self.get_summary(text, max_words) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." - summaries.append(command) + summaries = [summary, command] msg = "\n".join(summaries) logger.info(f"title ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) From 3112680324a2ba42ecf39b31796d14c605509848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:30:19 +0800 Subject: [PATCH 183/378] fixbug: summary too long --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 4764b6aad..b1d8aaa4a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -260,7 +260,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_context_title(self, text: str, max_words=5) -> str: """Generate text title""" - summary = await self.get_summary(text, max_words) + summary = await self.get_summary(text, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 264799541155c6ff59727a15e55b7b2ec5d4582c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:38:49 +0800 Subject: [PATCH 184/378] fixbug: summary too long --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b1d8aaa4a..b2a0faca5 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(text) < max_token_count: return await self._get_summary(text, max_words=max_words) - text_windows = self.split_texts(text, window_size=max_token_count - max_words) + padding_size = 20 if max_token_count > 20 else 0 + text_windows = self.split_texts(text, window_size=max_token_count - padding_size) summaries = [] for ws in text_windows: response = await self._get_summary(ws, max_words=max_words) From 5980b08c80451740ad5c3c3e057a146dcffb8694 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 21:48:23 +0800 Subject: [PATCH 185/378] fixbug: summary too long --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index fdd697b59..c707cb6f1 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -123,7 +123,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_context_title(history_text, max_token_count_per_ask=1000, max_words=500) + history_summary = await self._llm.get_summary(history_text, max_words=500) if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk From bf6388d1717cab8bd78671dbe0c13d7e421e7298 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 2 Sep 2023 22:28:56 +0800 Subject: [PATCH 186/378] =?UTF-8?q?fixbug:=20fix=20=E5=9B=BE=E7=89=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/learn/text_to_image.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/learn/text_to_image.py b/metagpt/learn/text_to_image.py index dd85cf617..23c2bddad 100644 --- a/metagpt/learn/text_to_image.py +++ b/metagpt/learn/text_to_image.py @@ -35,5 +35,5 @@ async def text_to_image(text, size_type: str = "512x512", openai_api_key="", mod s3 = S3() url = await s3.cache(data=base64_data, file_ext=".png", format=BASE64_FORMAT) if url: - return f"[{text}]({url})" + return f"![{text}]({url})" return image_declaration + base64_data if base64_data else "" From 69ef295b26f185f12c9e8bb05d79695425d01df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 12:11:37 +0800 Subject: [PATCH 187/378] fixbug: skill name --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c707cb6f1..0bce4a3f9 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -57,7 +57,9 @@ class Assistant(Role): prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: text_to_image\n" + prompt += ( + f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + ) prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" From 5079add5f829b05f193f91bb9dce121cf29e6517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 12:55:25 +0800 Subject: [PATCH 188/378] debug: +code --- metagpt/actions/skill_action.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 3ef0087fc..6bce2a634 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -7,8 +7,8 @@ @Desc : Call learned skill """ from __future__ import annotations + import ast -import importlib import traceback from metagpt.actions import Action, ActionOutput @@ -18,7 +18,7 @@ from metagpt.logs import logger class ArgumentsParingAction(Action): def __init__(self, last_talk: str, skill: Skill, context=None, llm=None, **kwargs): - super(ArgumentsParingAction, self).__init__(name='', context=context, llm=llm) + super(ArgumentsParingAction, self).__init__(name="", context=context, llm=llm) self.skill = skill self.ask = last_talk self.rsp = None @@ -56,10 +56,10 @@ class ArgumentsParingAction(Action): return None begin_ix = txt.find(prefix) end_ix = txt.rfind(")") - args_txt = txt[begin_ix + len(prefix): end_ix] + args_txt = txt[begin_ix + len(prefix) : end_ix] logger.info(args_txt) fake_expression = f"dict({args_txt})" - parsed_expression = ast.parse(fake_expression, mode='eval') + parsed_expression = ast.parse(fake_expression, mode="eval") args = {} for keyword in parsed_expression.body.keywords: key = keyword.arg @@ -70,7 +70,7 @@ class ArgumentsParingAction(Action): class SkillAction(Action): def __init__(self, skill: Skill, args: dict, context=None, llm=None, **kwargs): - super(SkillAction, self).__init__(name='', context=context, llm=llm) + super(SkillAction, self).__init__(name="", context=context, llm=llm) self._skill = skill self._args = args self.rsp = None @@ -86,17 +86,21 @@ class SkillAction(Action): @staticmethod async def find_and_call_function(function_name, args, **kwargs): + from metagpt.learn import text_to_speech + try: - module = importlib.import_module("metagpt.learn") - function = getattr(module, function_name) - # 调用函数并返回结果 - result = await function(**args, **kwargs) + result = await text_to_speech(**args, **kwargs) + # module = importlib.import_module("metagpt.learn") + # function = getattr(module, function_name) + # # 调用函数并返回结果 + # result = await function(**args, **kwargs) return result except (ModuleNotFoundError, AttributeError): logger.error(f"{function_name} not found") return None -if __name__ == '__main__': - ArgumentsParingAction.parse_arguments(skill_name="text_to_image", - txt='`text_to_image(text="Draw an apple", size_type="512x512")`') +if __name__ == "__main__": + ArgumentsParingAction.parse_arguments( + skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`' + ) From 04b348e92967d6a99ca0425c6aad1f3b34485e30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 13:31:52 +0800 Subject: [PATCH 189/378] feat: archive --- metagpt/actions/skill_action.py | 36 ++++++++++++++++++++++++--------- metagpt/learn/text_to_speech.py | 10 ++++----- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 6bce2a634..660d785ff 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -9,10 +9,14 @@ from __future__ import annotations import ast +import asyncio +import importlib import traceback +from copy import deepcopy from metagpt.actions import Action, ActionOutput -from metagpt.learn.skill_loader import Skill +from metagpt.config import CONFIG +from metagpt.learn.skill_loader import Returns, Skill from metagpt.logs import logger @@ -77,8 +81,13 @@ class SkillAction(Action): async def run(self, *args, **kwargs) -> str | ActionOutput | None: """Run action""" + options = deepcopy(kwargs) + if self._args: + for k in self._args.keys(): + if k in options: + options.pop(k) try: - self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **kwargs) + self.rsp = await self.find_and_call_function(self._skill.name, args=self._args, **options) except Exception as e: logger.exception(f"{e}, traceback:{traceback.format_exc()}") self.rsp = f"Error: {e}" @@ -86,14 +95,11 @@ class SkillAction(Action): @staticmethod async def find_and_call_function(function_name, args, **kwargs): - from metagpt.learn import text_to_speech - try: - result = await text_to_speech(**args, **kwargs) - # module = importlib.import_module("metagpt.learn") - # function = getattr(module, function_name) - # # 调用函数并返回结果 - # result = await function(**args, **kwargs) + module = importlib.import_module("metagpt.learn") + function = getattr(module, function_name) + # 调用函数并返回结果 + result = await function(**args, **kwargs) return result except (ModuleNotFoundError, AttributeError): logger.error(f"{function_name} not found") @@ -104,3 +110,15 @@ if __name__ == "__main__": ArgumentsParingAction.parse_arguments( skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`' ) + CONFIG.set_context({}) + args = {"text": "hello world", "role": "Girl"} + action = SkillAction( + skill=Skill( + name="text_to_speech", description="", id="", arguments={}, examples=[], returns=Returns(type="string") + ), + args=args, + ) + loop = asyncio.new_event_loop() + t = loop.create_task(action.run()) + r = loop.run_until_complete(t) + print(r) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 819da2364..eaceb3313 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -9,9 +9,7 @@ import openai from metagpt.config import CONFIG -from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts -from metagpt.utils.s3 import S3 async def text_to_speech( @@ -40,10 +38,10 @@ async def text_to_speech( audio_declaration = "data:audio/wav;base64," if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) - s3 = S3() - url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) - if url: - return f"[{text}]({url})" + # s3 = S3() + # url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) + # if url: + # return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data raise openai.error.InvalidRequestError("缺少必要的参数") From 0dddab18b44a053ef2d2206bfbf669750de0df3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 13:40:05 +0800 Subject: [PATCH 190/378] fixbug: no param --- metagpt/learn/text_to_speech.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index eaceb3313..691aa7f6a 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -44,4 +44,4 @@ async def text_to_speech( # return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data - raise openai.error.InvalidRequestError("缺少必要的参数") + raise openai.error.InvalidRequestError(message="AZURE_TTS_SUBSCRIPTION_KEY and AZURE_TTS_REGION error", param={}) From ef98ad4043b377037dd38d2aec1354bb7ea7be03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 13:46:23 +0800 Subject: [PATCH 191/378] fixbug: no param --- metagpt/actions/skill_action.py | 16 +--------------- metagpt/learn/text_to_speech.py | 10 ++++++---- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 660d785ff..758591fdd 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -9,14 +9,12 @@ from __future__ import annotations import ast -import asyncio import importlib import traceback from copy import deepcopy from metagpt.actions import Action, ActionOutput -from metagpt.config import CONFIG -from metagpt.learn.skill_loader import Returns, Skill +from metagpt.learn.skill_loader import Skill from metagpt.logs import logger @@ -110,15 +108,3 @@ if __name__ == "__main__": ArgumentsParingAction.parse_arguments( skill_name="text_to_image", txt='`text_to_image(text="Draw an apple", size_type="512x512")`' ) - CONFIG.set_context({}) - args = {"text": "hello world", "role": "Girl"} - action = SkillAction( - skill=Skill( - name="text_to_speech", description="", id="", arguments={}, examples=[], returns=Returns(type="string") - ), - args=args, - ) - loop = asyncio.new_event_loop() - t = loop.create_task(action.run()) - r = loop.run_until_complete(t) - print(r) diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 691aa7f6a..81bc8512b 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -9,7 +9,9 @@ import openai from metagpt.config import CONFIG +from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.utils.s3 import S3 async def text_to_speech( @@ -38,10 +40,10 @@ async def text_to_speech( audio_declaration = "data:audio/wav;base64," if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) - # s3 = S3() - # url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) - # if url: - # return f"[{text}]({url})" + s3 = S3() + url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) + if url: + return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data raise openai.error.InvalidRequestError(message="AZURE_TTS_SUBSCRIPTION_KEY and AZURE_TTS_REGION error", param={}) From 2856acb3f343b7a4d14643c52352ed2da6bc3119 Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Sun, 3 Sep 2023 17:22:36 +0800 Subject: [PATCH 192/378] faiss search kwargs --- metagpt/document_store/faiss_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 051bc2507..b034f40b2 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -51,7 +51,7 @@ class FaissStore(LocalStore): store.index = index def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs): - rsp = self.store.similarity_search(query, k=k) + rsp = self.store.similarity_search(query, k=k, **kwargs) logger.debug(rsp) if expand_cols: return str(sep.join([f"{x.page_content}: {x.metadata}" for x in rsp])) From 2285f0566ed214fd4cc4636f4da258b138931258 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 19:38:01 +0800 Subject: [PATCH 193/378] refactor: prompt --- metagpt/actions/talk_action.py | 53 ++++++++++++++++++++++++---------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 4eed0d4f8..12b9da030 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -27,22 +27,16 @@ class TalkAction(Action): @property def prompt(self): - prompt = "" - if CONFIG.agent_description: - prompt = ( - f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " - f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " - f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" - ) - prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" - prompt += f"{self._history_summary}\n\n" - if self._history_summary != "": - prompt += "According to the historical conversation above, " - language = CONFIG.language or DEFAULT_LANGUAGE - prompt += ( - f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " - f"{self._talk}" - ) + kvs = { + "{role}": CONFIG.agent_description or "", + "{history}": self._history_summary or "", + "{knowledge}": self._knowledge or "", + "{language}": CONFIG.language or DEFAULT_LANGUAGE, + "{ask}": self._talk, + } + prompt = TalkAction.__FORMATION__ + for k, v in kvs.items(): + prompt = prompt.replace(k, v) return prompt async def run(self, *args, **kwargs) -> ActionOutput: @@ -52,3 +46,30 @@ class TalkAction(Action): logger.info(rsp) self._rsp = ActionOutput(content=rsp) return self._rsp + + __FORMATION__ = """Formation: "Capacity and role" defines the role you are currently playing; + "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; + "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; + "Statement" defines the work detail you need to complete at this stage; + "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "Constraint" defines the conditions that your responses must comply with. + +Capacity and role: {role} +Statement: Your responses should align with the role-play agreement, maintaining the + character's persona and habits. When faced with unrelated questions, playfully decline to answer without revealing + your AI nature to preserve the character's image. Statement: the answers must follow the Markdown format. + +[HISTORY_BEGIN] +{history} +[HISTORY_END] + +[KNOWLEDGE_BEGIN] +{knowledge} +[KNOWLEDGE_END] + +Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in + {language}, and the answers must follow the Markdown format. + + [ASK_BEGIN] + {ask} + [ASK_END]""" From d6ffa4906f71205ec4a358152eb1ba81fffe60f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 19:55:53 +0800 Subject: [PATCH 194/378] refactor: prompt --- metagpt/actions/talk_action.py | 39 ++++++++++++++++++++++++++++------ 1 file changed, 33 insertions(+), 6 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 12b9da030..fead3c8b9 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -34,7 +34,7 @@ class TalkAction(Action): "{language}": CONFIG.language or DEFAULT_LANGUAGE, "{ask}": self._talk, } - prompt = TalkAction.__FORMATION__ + prompt = TalkAction.__FORMATION_LOOSE__ for k, v in kvs.items(): prompt = prompt.replace(k, v) return prompt @@ -57,7 +57,34 @@ class TalkAction(Action): Capacity and role: {role} Statement: Your responses should align with the role-play agreement, maintaining the character's persona and habits. When faced with unrelated questions, playfully decline to answer without revealing - your AI nature to preserve the character's image. Statement: the answers must follow the Markdown format. + your AI nature to preserve the character's image. + +[HISTORY_BEGIN] +{history} +[HISTORY_END] + +[KNOWLEDGE_BEGIN] +{knowledge} +[KNOWLEDGE_END] + +Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in +{language}, and the answers must follow the Markdown format, excluding any tag likes "[HISTORY_BEGIN]", +"[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" + +[ASK_BEGIN] +{ask} +[ASK_END]""" + + __FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing; + "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; + "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; + "Statement" defines the work detail you need to complete at this stage; + "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "Constraint" defines the conditions that your responses must comply with. + +Capacity and role: {role} +Statement: Your responses should maintaining the character's persona and habits. When faced with unrelated questions +, playfully decline to answer without revealing your AI nature to preserve the character's image. [HISTORY_BEGIN] {history} @@ -69,7 +96,7 @@ Statement: Your responses should align with the role-play agreement, maintaining Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in {language}, and the answers must follow the Markdown format. - - [ASK_BEGIN] - {ask} - [ASK_END]""" + +[ASK_BEGIN] +{ask} +[ASK_END]""" From b5c149f22507ffe139ca9333c50934af16a36611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sun, 3 Sep 2023 20:02:24 +0800 Subject: [PATCH 195/378] refactor: prompt --- metagpt/actions/talk_action.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index fead3c8b9..2a04fb9c8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -67,9 +67,10 @@ Statement: Your responses should align with the role-play agreement, maintaining {knowledge} [KNOWLEDGE_END] -Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in -{language}, and the answers must follow the Markdown format, excluding any tag likes "[HISTORY_BEGIN]", -"[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" +Statement: If the information is insufficient, you can search in the historical conversation or knowledge. +Statement: Answer the following questions in {language}, and the answers must follow the Markdown format + , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" + , "[ASK_END]" [ASK_BEGIN] {ask} @@ -94,8 +95,10 @@ Statement: Your responses should maintaining the character's persona and habits. {knowledge} [KNOWLEDGE_END] -Statement: According to the historical conversation and knowledge above if helpful, Answer the following questions in - {language}, and the answers must follow the Markdown format. +Statement: If the information is insufficient, you can search in the historical conversation or knowledge. +Statement: Answer the following questions in {language}, and the answers must follow the Markdown format + , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" + , "[ASK_END]" [ASK_BEGIN] {ask} From b036b5d22ee17c59f0d01124dea98c34e8ff0a99 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sun, 3 Sep 2023 22:22:26 +0800 Subject: [PATCH 196/378] remove openai global settings --- metagpt/provider/openai_api.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index b2a0faca5..844cd4c1c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -77,21 +77,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ def __init__(self): - self.__init_openai(CONFIG) self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False + self.rpm = int(CONFIG.get("RPM", 10)) RateLimiter.__init__(self, rpm=self.rpm) - def __init_openai(self, config): - openai.api_key = config.openai_api_key - if config.openai_api_base: - openai.api_base = config.openai_api_base - if config.openai_api_type: - openai.api_type = config.openai_api_type - openai.api_version = config.openai_api_version - self.rpm = int(config.get("RPM", 10)) - async def _achat_completion_stream(self, messages: list[dict]) -> str: response = await self.async_retry_call( openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True @@ -133,6 +124,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): "temperature": 0.3, } kwargs["timeout"] = 3 + kwargs["api_base"] = CONFIG.openai_api_base + kwargs["api_key"] = CONFIG.openai_api_key + kwargs["api_type"] = CONFIG.openai_api_type + kwargs["api_version"] = CONFIG.openai_api_version return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: From e06aa62ac4dcd5ed4ec401f16ae34ecd4f178034 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 10:11:19 +0800 Subject: [PATCH 197/378] refactor: prompt --- metagpt/actions/talk_action.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 2a04fb9c8..526d921f8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -27,6 +27,26 @@ class TalkAction(Action): @property def prompt(self): + prompt = "" + if CONFIG.agent_description: + prompt = ( + f"You are {CONFIG.agent_description}. Your responses should align with the role-play agreement, " + f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " + f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" + ) + prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt += f"{self._history_summary}\n\n" + if self._history_summary != "": + prompt += "According to the historical conversation above, " + language = CONFIG.language or DEFAULT_LANGUAGE + prompt += ( + f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " + f"{self._talk}" + ) + return prompt + + @property + def prompt_new(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 63594cd8fd924ca3aff0153354fd78e5e415b507 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 10:12:10 +0800 Subject: [PATCH 198/378] refactor: prompt --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 526d921f8..83504b62d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -46,7 +46,7 @@ class TalkAction(Action): return prompt @property - def prompt_new(self): + def formation_prompt(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 87f4c22b6050ea7b951498b03d3cc9149dc54fb9 Mon Sep 17 00:00:00 2001 From: Stitch-z <284618289@qq.com> Date: Mon, 4 Sep 2023 10:48:48 +0800 Subject: [PATCH 199/378] update: aioboto3 client async open file --- metagpt/utils/s3.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 74c3f1654..96b457972 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -44,8 +44,9 @@ class S3: """ try: async with self.session.client(**self.auth_config) as client: - with open(local_path, "rb") as file: - await client.put_object(Body=file, Bucket=bucket, Key=object_name) + async with aiofiles.open(local_path, mode="rb") as reader: + body = await reader.read() + await client.put_object(Body=body, Bucket=bucket, Key=object_name) logger.info(f"Successfully uploaded the file to path {object_name} in bucket {bucket} of s3.") except Exception as e: logger.error(f"Failed to upload the file to path {object_name} in bucket {bucket} of s3: {e}") @@ -119,12 +120,12 @@ class S3: async with self.session.client(**self.auth_config) as client: s3_object = await client.get_object(Bucket=bucket, Key=object_name) stream = s3_object["Body"] - with open(local_path, "wb") as local_file: + async with aiofiles.open(local_path, mode="wb") as writer: while True: file_data = await stream.read(chunk_size) if not file_data: break - local_file.write(file_data) + await writer.write(file_data) except Exception as e: logger.error(f"Failed to download the file from S3: {e}") raise e From d4878f23a0042bf983c1fef8947c649f7d4f4878 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Mon, 4 Sep 2023 10:50:21 +0800 Subject: [PATCH 200/378] =?UTF-8?q?update:=E4=BF=AE=E6=94=B9get=5Fsummary?= =?UTF-8?q?=EF=BC=8C=E5=8A=A0=E4=B8=8A=E6=98=AF=E5=90=A6=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E8=AF=AD=E8=A8=80=E7=9A=84=E9=85=8D=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/openai_api.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 844cd4c1c..26929575c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -221,18 +221,18 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=200): + async def get_summary(self, text: str, max_words=200, keep_language: bool = False): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 while max_count > 0: if len(text) < max_token_count: - return await self._get_summary(text, max_words=max_words) + return await self._get_summary(text=text, max_words=max_words,keep_language=keep_language) padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) summaries = [] for ws in text_windows: - response = await self._get_summary(ws, max_words=max_words) + response = await self._get_summary(text=ws, max_words=max_words,keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] @@ -243,11 +243,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): max_count -= 1 # safeguard raise openai.error.InvalidRequestError("text too long") - async def _get_summary(self, text: str, max_words=20): + async def _get_summary(self, text: str, max_words=20, keep_language: bool = False): """Generate text summary""" if len(text) < max_words: return text - command = f"Translate the above content into a summary of less than {max_words} words." + if keep_language: + command = f".Translate the above content into a summary of less than {max_words} words in language of the content." + else: + command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command logger.info(f"summary ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) From 2f95a8a2000aee5e1aa07a29259a81cdd0c800f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 11:42:14 +0800 Subject: [PATCH 201/378] feat: +config --- config/config.yaml | 9 ++++++++- metagpt/utils/redis.py | 0 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 metagpt/utils/redis.py diff --git a/config/config.yaml b/config/config.yaml index 7c3d212f6..765a74b8a 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -83,4 +83,11 @@ MODEL_FOR_RESEARCHER_REPORT: gpt-3.5-turbo-16k S3: access_key: "YOUR_S3_ACCESS_KEY" secret_key: "YOUR_S3_SECRET_KEY" - endpoint_url: "YOUR_S3_ENDPOINT_URL" \ No newline at end of file + endpoint_url: "YOUR_S3_ENDPOINT_URL" + +### Redis config +REDIS: + host: "YOUR_REDIS_HOST" + port: YOUR_REDIS_PORT, int + password: "YOUR_REDIS_PASSWORD" + db: YOUR_REDIS_DB_INDEX, int \ No newline at end of file diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py new file mode 100644 index 000000000..e69de29bb From 9cc85d631ad15fe369f1cd647a4071ca31bd6a94 Mon Sep 17 00:00:00 2001 From: zhanglei Date: Mon, 4 Sep 2023 11:50:22 +0800 Subject: [PATCH 202/378] =?UTF-8?q?update:=E4=BF=AE=E6=94=B9get=5Fsummary?= =?UTF-8?q?=EF=BC=8C=E5=8A=A0=E4=B8=8A=E6=98=AF=E5=90=A6=E4=BF=9D=E6=8C=81?= =?UTF-8?q?=E8=AF=AD=E8=A8=80=E7=9A=84=E9=85=8D=E7=BD=AE,=E5=BC=BA?= =?UTF-8?q?=E8=B0=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 26929575c..5c11ed7a6 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -248,7 +248,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): if len(text) < max_words: return text if keep_language: - command = f".Translate the above content into a summary of less than {max_words} words in language of the content." + command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly." else: command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command From 96f833cf8fafcea3555efd5871bea2ed2364647f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 12:47:45 +0800 Subject: [PATCH 203/378] feat: +redis --- metagpt/memory/brain_memory.py | 34 ++++-- metagpt/utils/redis.py | 198 +++++++++++++++++++++++++++++++++ requirements.txt | 3 +- 3 files changed, 222 insertions(+), 13 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a5a3dbfc7..275cd14df 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -6,7 +6,7 @@ @File : brain_memory.py @Desc : Support memory for multiple tasks and multiple mainlines. """ - +import hashlib from enum import Enum from typing import Dict, List @@ -28,6 +28,10 @@ class BrainMemory(pydantic.BaseModel): stack: List[Dict] = [] solution: List[Dict] = [] knowledge: List[Dict] = [] + # If the fingerprint of the history text is found in the `historical_summary_fingerprint`, + # it indicates that the text has already been incorporated into the `history summary`. + historical_summary_fingerprint: List[str] = [] + historical_summary: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -58,17 +62,19 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) def move_to_solution(self, history_summary): - """放入solution队列，以备后续长程检索。目前还未加此功能，先用history_summary顶替""" - if len(self.history) < 2: - return - msgs = self.history[:-1] - self.solution.extend(msgs) - if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): - self.solution.append(self.history[-1]) - self.history = [] - else: - self.history = self.history[-1:] - self.history.insert(0, Message(content="RESOLVED: " + history_summary)) + """Put it in the solution queue for future long-term retrieval. + This functionality hasn't been added yet, so use the history summary as a temporary substitute for now.""" + pass + # if len(self.history) < 2: + # return + # msgs = self.history[:-1] + # self.solution.extend(msgs) + # if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): + # self.solution.append(self.history[-1]) + # self.history = [] + # else: + # self.history = self.history[-1:] + # self.history.insert(0, Message(content="RESOLVED: " + history_summary)) @property def last_talk(self): @@ -78,3 +84,7 @@ class BrainMemory(pydantic.BaseModel): if not last_msg.is_contain(MessageType.Talk.value): return None return last_msg.content + + @staticmethod + def get_md5(text: str) -> str: + return hashlib.md5(text.encode()).hexdigest() diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index e69de29bb..f2ae3222a 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -0,0 +1,198 @@ +# !/usr/bin/python3 +# -*- coding: utf-8 -*- +# @Author: Hui +# @Desc: { redis client } +# @Date: 2022/11/28 10:12 +import json +from datetime import timedelta +from enum import Enum +from typing import Awaitable, Callable, Optional, Union + +from redis import asyncio as aioredis + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class RedisTypeEnum(Enum): + """Redis 数据类型""" + + String = "String" + List = "List" + Hash = "Hash" + Set = "Set" + ZSet = "ZSet" + + +def make_url( + dialect: str, + *, + user: Optional[str] = None, + password: Optional[str] = None, + host: Optional[str] = None, + port: Optional[Union[str, int]] = None, + name: Optional[Union[str, int]] = None, +) -> str: + url_parts = [f"{dialect}://"] + if user or password: + if user: + url_parts.append(user) + if password: + url_parts.append(f":{password}") + url_parts.append("@") + + if not host and not dialect.startswith("sqlite"): + host = "127.0.0.1" + + if host: + url_parts.append(f"{host}") + if port: + url_parts.append(f":{port}") + + # 比如redis可能传入0 + if name is not None: + url_parts.append(f"/{name}") + return "".join(url_parts) + + +class RedisAsyncClient(aioredis.Redis): + """异步的客户端 + 例子:: + + rdb = RedisAsyncClient() + print(rdb.url) + + Args: + host: 服务器地址 + port: 服务器端口 + user: 用户名 + db: 数据库 + password: 密码 + decode_responses: 字符串输入被编码成utf8存储在Redis里了，而取出来的时候还是被编码后的bytes，需要显示的decode才能变成字符串 + health_check_interval: 定时检测连接，防止出现ConnectionErrors (104, Connection reset by peer) + """ + + def __init__( + self, + host: str = "localhost", + port: int = 6379, + db: int = 0, + password: str = None, + decode_responses=True, + health_check_interval=10, + socket_connect_timeout=5, + retry_on_timeout=True, + socket_keepalive=True, + **kwargs, + ): + super().__init__( + host=host, + port=port, + db=db, + password=password, + decode_responses=decode_responses, + health_check_interval=health_check_interval, + socket_connect_timeout=socket_connect_timeout, + retry_on_timeout=retry_on_timeout, + socket_keepalive=socket_keepalive, + **kwargs, + ) + self.url = make_url("redis", host=host, port=port, name=db, password=password) + + +class RedisCacheInfo(object): + """统一缓存信息类""" + + def __init__(self, key, timeout: Union[int, timedelta] = timedelta(seconds=60), data_type=RedisTypeEnum.String): + """ + 缓存信息类初始化 + Args: + key: 缓存的key + timeout: 缓存过期时间, 单位秒 + data_type: 缓存采用的数据结构 (不传并不影响，用于标记业务采用的是什么数据结构) + """ + self.key = key + self.timeout = timeout + self.data_type = data_type + + def __str__(self): + return f"cache key {self.key} timeout {self.timeout}s" + + +class RedisManager: + client: RedisAsyncClient = None + + @classmethod + def init_redis_conn(cls, host, port, password, db): + """初始化redis 连接""" + if cls.client is None: + cls.client = RedisAsyncClient(host=host, port=port, password=password, db=db) + + @classmethod + async def set_with_cache_info(cls, redis_cache_info: RedisCacheInfo, value): + """ + 根据 RedisCacheInfo 设置 Redis 缓存 + :param redis_cache_info: RedisCacheInfo缓存信息对象 + :param value: 缓存的值 + :return: + """ + await cls.client.setex(redis_cache_info.key, redis_cache_info.timeout, value) + + @classmethod + async def get_with_cache_info(cls, redis_cache_info: RedisCacheInfo): + """ + 根据 RedisCacheInfo 获取 Redis 缓存 + :param redis_cache_info: RedisCacheInfo 缓存信息对象 + :return: + """ + cache_info = await cls.client.get(redis_cache_info.key) + return cache_info + + @classmethod + async def del_with_cache_info(cls, redis_cache_info: RedisCacheInfo): + """ + 根据 RedisCacheInfo 删除 Redis 缓存 + :param redis_cache_info: RedisCacheInfo缓存信息对象 + :return: + """ + await cls.client.delete(redis_cache_info.key) + + @staticmethod + async def get_or_set_cache(cache_info: RedisCacheInfo, fetch_data_func: Callable[[], Awaitable[dict]]) -> dict: + """ + 获取缓存数据，如果缓存不存在，则从提供的函数中获取并设置缓存 + 当前版本仅支持 json 形式的 string 格式数据 + """ + + serialized_data = await RedisManager.get_with_cache_info(cache_info) + + if serialized_data: + return json.loads(serialized_data) + + data = await fetch_data_func() + try: + serialized_data = json.dumps(data) + await RedisManager.set_with_cache_info(cache_info, serialized_data) + except Exception as e: + logger.warning(f"数据 {data} 通过 json 进行序列化缓存失败:{e}") + + return data + + @classmethod + def is_valid(cls): + return cls.client is not None + + +class Redis: + def __init__(self): + self._config = CONFIG.REDIS + if not self._config: + return + try: + host = self._config["host"] + port = int(self._config["port"]) + pwd = self._config["password"] + db = int(self._config["db"]) + RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db) + except Exception as e: + logger.warning(f"Redis initialization has failed:{e}") diff --git a/requirements.txt b/requirements.txt index 5daf710c7..588b29e0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,4 +41,5 @@ qdrant-client==1.4.0 connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 -aioboto3~=11.3.0 \ No newline at end of file +aioboto3~=11.3.0 +redis==4.3.5 \ No newline at end of file From 0ffd3db9473eda5e2172e8bc826638feddb987cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 13:21:29 +0800 Subject: [PATCH 204/378] feat: +redis --- metagpt/const.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/const.py b/metagpt/const.py index fbc2c928a..e9fa118d7 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -57,3 +57,6 @@ METAGPT_API_VERSION = "METAGPT_API_VERSION" # format BASE64_FORMAT = "base64" + +# REDIS +REDIS_KEY = "REDIS_KEY" From cce76df319ed5174d8a1aca88d498354856b741f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:03:20 +0800 Subject: [PATCH 205/378] feat: +redis --- metagpt/memory/brain_memory.py | 21 +++++++++++++++++++++ metagpt/utils/redis.py | 16 ++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 275cd14df..619a9e1f3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -7,12 +7,14 @@ @Desc : Support memory for multiple tasks and multiple mainlines. """ import hashlib +import json from enum import Enum from typing import Dict, List import pydantic from metagpt import Message +from metagpt.utils.redis import Redis class MessageType(Enum): @@ -32,6 +34,7 @@ class BrainMemory(pydantic.BaseModel): # it indicates that the text has already been incorporated into the `history summary`. historical_summary_fingerprint: List[str] = [] historical_summary: str = "" + last_history_id: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -88,3 +91,21 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def get_md5(text: str) -> str: return hashlib.md5(text.encode()).hexdigest() + + @staticmethod + async def loads(redis_key: str) -> "BrainMemory": + redis = Redis() + if not redis.is_valid() or not redis_key: + return False + v = await redis.get(key=redis_key) + if not v: + data = json.loads(v) + return BrainMemory(**data) + return None + + async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60): + redis = Redis() + if not redis.is_valid() or not redis_key: + return False + v = self.json() + await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index f2ae3222a..ce9d1bc8e 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -196,3 +196,19 @@ class Redis: RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db) except Exception as e: logger.warning(f"Redis initialization has failed:{e}") + + def is_valid(self): + return RedisManager.is_valid() + + async def get(self, key: str) -> str: + if not self.is_valid() or not key: + return None + v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key)) + return v + + async def set(self, key: str, data: str, timeout_sec: int): + if not self.is_valid() or not key: + return + await RedisManager.set_with_cache_info( + redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data + ) From 41e90b4f483da8d734fb7497975e499330f46e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:12:17 +0800 Subject: [PATCH 206/378] feat: +redis --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 619a9e1f3..baad76562 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -96,12 +96,12 @@ class BrainMemory(pydantic.BaseModel): async def loads(redis_key: str) -> "BrainMemory": redis = Redis() if not redis.is_valid() or not redis_key: - return False + return BrainMemory() v = await redis.get(key=redis_key) if not v: data = json.loads(v) return BrainMemory(**data) - return None + return BrainMemory() async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60): redis = Redis() From 26c4ed6e2245ecd19423cadc0faf697241170528 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:38:35 +0800 Subject: [PATCH 207/378] feat: +code --- metagpt/memory/brain_memory.py | 8 ++++---- metagpt/utils/redis.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index baad76562..3b27c2a94 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -93,8 +93,8 @@ class BrainMemory(pydantic.BaseModel): return hashlib.md5(text.encode()).hexdigest() @staticmethod - async def loads(redis_key: str) -> "BrainMemory": - redis = Redis() + async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": + redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) @@ -103,8 +103,8 @@ class BrainMemory(pydantic.BaseModel): return BrainMemory(**data) return BrainMemory() - async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60): - redis = Redis() + async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None): + redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: return False v = self.json() diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index ce9d1bc8e..7d1d88fbd 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -6,7 +6,7 @@ import json from datetime import timedelta from enum import Enum -from typing import Awaitable, Callable, Optional, Union +from typing import Awaitable, Callable, Dict, Optional, Union from redis import asyncio as aioredis @@ -184,8 +184,8 @@ class RedisManager: class Redis: - def __init__(self): - self._config = CONFIG.REDIS + def __init__(self, conf: Dict = None): + self._config = conf or CONFIG.REDIS if not self._config: return try: From d6130c2d99361d02c7a68cf9384d7ae3660f8d25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:50:54 +0800 Subject: [PATCH 208/378] feat: +to_redis_key --- metagpt/memory/brain_memory.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 3b27c2a94..faf7693ad 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -109,3 +109,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + + @staticmethod + def to_redis_key(prefix: str, user_id: str, chat_id: str): + return f"{prefix}:{chat_id}:{user_id}" From 0e717a0537c854b7fdd7674c4a7326898e33092f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:54:40 +0800 Subject: [PATCH 209/378] feat: +to_redis_key --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 765a74b8a..5c8dea03e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -90,4 +90,4 @@ REDIS: host: "YOUR_REDIS_HOST" port: YOUR_REDIS_PORT, int password: "YOUR_REDIS_PASSWORD" - db: YOUR_REDIS_DB_INDEX, int \ No newline at end of file + db: "YOUR_REDIS_DB_INDEX, str, 0-based" \ No newline at end of file From 308f83c82c4442d42613d642c5080a6d07a052a1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 14:55:26 +0800 Subject: [PATCH 210/378] feat: +to_redis_key --- metagpt/utils/redis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index 7d1d88fbd..b94eee8e2 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -192,7 +192,7 @@ class Redis: host = self._config["host"] port = int(self._config["port"]) pwd = self._config["password"] - db = int(self._config["db"]) + db = self._config["db"] RedisManager.init_redis_conn(host=host, port=port, password=pwd, db=db) except Exception as e: logger.warning(f"Redis initialization has failed:{e}") From 0a494171fa71b789f685c676ea6b7612c4785bb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 15:30:48 +0800 Subject: [PATCH 211/378] fixbug: prerequisite --- metagpt/roles/role.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index 2f0f713f8..b1ace19fa 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -97,8 +97,9 @@ class RoleContext(BaseModel): def prerequisite(self): """Retrieve information with `prerequisite` tag""" if self.memory and hasattr(self.memory, "get_by_tags"): - return self.memory.get_by_tags([MessageTag.Prerequisite.value]) - return "" + vv = self.memory.get_by_tags([MessageTag.Prerequisite.value]) + return vv[-1:] if len(vv) > 1 else vv + return [] class Role: From fb6bb4b69210909dbf842e83f6fd2277bb61990c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 15:39:11 +0800 Subject: [PATCH 212/378] feat: is dirty --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index faf7693ad..8ae7ed959 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -35,14 +35,17 @@ class BrainMemory(pydantic.BaseModel): historical_summary_fingerprint: List[str] = [] historical_summary: str = "" last_history_id: str = "" + is_dirty: bool = False def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) self.history.append(msg.dict()) + self.is_dirty = True def add_answer(self, msg: Message): msg.add_tag(MessageType.Answer.value) self.history.append(msg.dict()) + self.is_dirty = True def get_knowledge(self) -> str: texts = [Message(**m).content for m in self.knowledge] From 82c7fd94fd9ff500eecdc3fcbd805301178feee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 15:50:17 +0800 Subject: [PATCH 213/378] feat: is dirty --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 8ae7ed959..a925474b7 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -101,7 +101,7 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) - if not v: + if v: data = json.loads(v) return BrainMemory(**data) return BrainMemory() From 88419224586ec683db92ae83e4b4aad35bfb5d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 16:05:01 +0800 Subject: [PATCH 214/378] feat: +cache --- metagpt/memory/brain_memory.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a925474b7..8b1b31aae 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -103,7 +103,9 @@ class BrainMemory(pydantic.BaseModel): v = await redis.get(key=redis_key) if v: data = json.loads(v) - return BrainMemory(**data) + bm = BrainMemory(**data) + bm.is_dirty = False + return bm return BrainMemory() async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None): @@ -112,6 +114,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + self.is_dirty = False @staticmethod def to_redis_key(prefix: str, user_id: str, chat_id: str): From c4a0bd14385f529cb441c7e527baf554d2d74601 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 16:40:40 +0800 Subject: [PATCH 215/378] fixbug: tags --- metagpt/schema.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/metagpt/schema.py b/metagpt/schema.py index ce08455fc..987fccef2 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -10,7 +10,7 @@ from __future__ import annotations from dataclasses import dataclass, field from enum import Enum -from typing import Type, TypedDict, Set, Optional, List +from typing import Optional, Set, Type, TypedDict from pydantic import BaseModel @@ -29,9 +29,10 @@ class RawMessage(TypedDict): @dataclass class Message: """list[: ]""" + content: str instruct_content: BaseModel = field(default=None) - role: str = field(default='user') # system / user / assistant + role: str = field(default="user") # system / user / assistant cause_by: Type["Action"] = field(default="") sent_from: str = field(default="") send_to: str = field(default="") @@ -45,10 +46,7 @@ class Message: return self.__str__() def to_dict(self) -> dict: - return { - "role": self.role, - "content": self.content - } + return {"role": self.role, "content": self.content} def add_tag(self, tag): if self.tags is None: @@ -64,7 +62,7 @@ class Message: """Determine whether the message contains tags.""" if not tags or not self.tags: return False - intersection = set(tags) & self.tags + intersection = set(tags) & set(self.tags) return len(intersection) > 0 def is_contain(self, tag): @@ -76,7 +74,7 @@ class Message: "instruct_content": self.instruct_content, "sent_from": self.sent_from, "send_to": self.send_to, - "tags": self.tags + "tags": self.tags, } m = {"content": self.content} @@ -89,39 +87,39 @@ class Message: @dataclass class UserMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ def __init__(self, content: str): - super().__init__(content, 'user') + super().__init__(content, "user") @dataclass class SystemMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ def __init__(self, content: str): - super().__init__(content, 'system') + super().__init__(content, "system") @dataclass class AIMessage(Message): """便于支持OpenAI的消息 - Facilitate support for OpenAI messages + Facilitate support for OpenAI messages """ def __init__(self, content: str): - super().__init__(content, 'assistant') + super().__init__(content, "assistant") -if __name__ == '__main__': - test_content = 'test_message' +if __name__ == "__main__": + test_content = "test_message" msgs = [ UserMessage(test_content), SystemMessage(test_content), AIMessage(test_content), - Message(test_content, role='QA') + Message(test_content, role="QA"), ] logger.info(msgs) From 230239b3e7fed3dabc21a9cf13568fde946cc1b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 16:46:11 +0800 Subject: [PATCH 216/378] feat: +cache --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 8b1b31aae..e487a696d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -55,7 +55,7 @@ class BrainMemory(pydantic.BaseModel): def history_text(self): if len(self.history) == 0: return "" - texts = [] + texts = [self.historical_summary] if self.historical_summary else [] for m in self.history[:-1]: if isinstance(m, Dict): t = Message(**m).content From 9220b131a433c8bf1f08a45053779832a7c275f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:08:19 +0800 Subject: [PATCH 217/378] feat: +cache --- metagpt/memory/brain_memory.py | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e487a696d..ed2955902 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -67,21 +67,6 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) - def move_to_solution(self, history_summary): - """Put it in the solution queue for future long-term retrieval. - This functionality hasn't been added yet, so use the history summary as a temporary substitute for now.""" - pass - # if len(self.history) < 2: - # return - # msgs = self.history[:-1] - # self.solution.extend(msgs) - # if not Message(**self.history[-1]).is_contain(MessageType.Talk.value): - # self.solution.append(self.history[-1]) - # self.history = [] - # else: - # self.history = self.history[-1:] - # self.history.insert(0, Message(content="RESOLVED: " + history_summary)) - @property def last_talk(self): if len(self.history) == 0: @@ -119,3 +104,12 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def to_redis_key(prefix: str, user_id: str, chat_id: str): return f"{prefix}:{chat_id}:{user_id}" + + async def set_history_summary(self, history_summary, redis_key, redis_conf): + if self.historical_summary == history_summary: + return + + self.historical_summary = history_summary + self.history = [] + await self.dumps(redis_key=redis_key, redis_conf=redis_conf) + self.is_dirty = False From f69f37bb0376b25f0d52eee2a68b72deac83391f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:09:02 +0800 Subject: [PATCH 218/378] feat: +cache --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 0bce4a3f9..9c80593f6 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -126,11 +126,13 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary(history_text, max_words=500) + await self.memory.set_history_summary( + history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS + ) if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk - self.memory.move_to_solution(history_summary) # Promptly clear memory after the issue is resolved. return last_talk @staticmethod From 32c604a002e78e924d43a732e4b4bd7e3bce1faf Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Mon, 4 Sep 2023 17:21:21 +0800 Subject: [PATCH 219/378] add llm.aask generator --- metagpt/provider/base_gpt_api.py | 4 ++-- metagpt/provider/openai_api.py | 34 +++++++++++++++++--------------- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index af0cf2ec0..7351e6916 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,13 +38,13 @@ class BaseGPTAPI(BaseChatbot): rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None) -> str: + async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, generator: bool = False) -> str: if system_msgs: message = self._system_msgs(system_msgs) + [self._user_msg(msg)] else: message = [self._default_system_msg(), self._user_msg(msg)] try: - rsp = await self.acompletion_text(message, stream=True) + rsp = await self.acompletion_text(message, stream=True, generator=generator) except Exception as e: logger.exception(f"{e}") logger.info(f"ask:{msg}, error:{e}") diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 5c11ed7a6..d0dd5b9d8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -87,22 +87,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): response = await self.async_retry_call( openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True ) - # create variables to collect the stream of chunks - collected_chunks = [] - collected_messages = [] # iterate through the stream of events async for chunk in response: - collected_chunks.append(chunk) # save the event response chunk_message = chunk["choices"][0]["delta"] # extract the message - collected_messages.append(chunk_message) # save the message if "content" in chunk_message: - print(chunk_message["content"], end="") - print() - - full_reply_content = "".join([m.get("content", "") for m in collected_messages]) - usage = self._calc_usage(messages, full_reply_content) - self._update_costs(usage) - return full_reply_content + yield chunk_message["content"] def _cons_kwargs(self, messages: list[dict]) -> dict: if CONFIG.openai_api_type == "azure": @@ -157,10 +146,23 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) - async def acompletion_text(self, messages: list[dict], stream=False) -> str: + async def acompletion_text(self, messages: list[dict], stream=False, generator: bool = False) -> str: """when streaming, print each token in place.""" if stream: - return await self._achat_completion_stream(messages) + resp = self._achat_completion_stream(messages) + if generator: + return resp + + collected_messages = [] + async for i in resp: + print(i, end="") + collected_messages.append(i) + + full_reply_content = "".join(collected_messages) + usage = self._calc_usage(messages, full_reply_content) + self._update_costs(usage) + return full_reply_content + rsp = await self._achat_completion(messages) return self.get_choice_text(rsp) @@ -226,13 +228,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): max_count = 100 while max_count > 0: if len(text) < max_token_count: - return await self._get_summary(text=text, max_words=max_words,keep_language=keep_language) + return await self._get_summary(text=text, max_words=max_words, keep_language=keep_language) padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=max_words,keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From ec8e455a59ff1d669ae7071dd8129ddef0abf45b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:47:33 +0800 Subject: [PATCH 220/378] feat: +cache --- metagpt/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 987fccef2..8f8e4030f 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -37,6 +37,7 @@ class Message: sent_from: str = field(default="") send_to: str = field(default="") tags: Optional[Set] = field(default=None) + id: str = None def __str__(self): # prefix = '-'.join([self.role, str(self.cause_by)]) From ebe5217f701157b1fba5e23effc194c6d3ce8560 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 17:58:40 +0800 Subject: [PATCH 221/378] feat: +cache --- metagpt/memory/brain_memory.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index ed2955902..8443d69d9 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -113,3 +113,10 @@ class BrainMemory(pydantic.BaseModel): self.history = [] await self.dumps(redis_key=redis_key, redis_conf=redis_conf) self.is_dirty = False + + def add_history(self, msg: Message): + if msg.id: + if int(msg.id) < int(self.last_history_id): + return + self.history.append(msg.dict()) + self.is_dirty = True From b5ea3c692f5988e2974c897cd21344ca40920e23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 18:07:03 +0800 Subject: [PATCH 222/378] feat: +cache --- metagpt/memory/brain_memory.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 8443d69d9..027297eb8 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -120,3 +120,9 @@ class BrainMemory(pydantic.BaseModel): return self.history.append(msg.dict()) self.is_dirty = True + + def exists(self, text) -> bool: + for m in reversed(self.history): + if m.get("content") == text: + return True + return False From 4d9cfe6f439387ef783b3fe9b38edc4e3efe250d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 18:57:16 +0800 Subject: [PATCH 223/378] feat: +cache --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 027297eb8..2ea8ac209 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -14,6 +14,7 @@ from typing import Dict, List import pydantic from metagpt import Message +from metagpt.logs import logger from metagpt.utils.redis import Redis @@ -86,6 +87,7 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) + logger.info(f"REDIS GET {redis_key} {v}") if v: data = json.loads(v) bm = BrainMemory(**data) @@ -99,6 +101,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + logger.info(f"REDIS SET {redis_key} {v}") self.is_dirty = False @staticmethod From 26e35d799db5ff32c4a935909a15eb71b763e51f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:02:54 +0800 Subject: [PATCH 224/378] feat: +cache --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 2ea8ac209..50c414c97 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -54,7 +54,7 @@ class BrainMemory(pydantic.BaseModel): @property def history_text(self): - if len(self.history) == 0: + if len(self.history) == 0 and not self.historical_summary: return "" texts = [self.historical_summary] if self.historical_summary else [] for m in self.history[:-1]: From 207ab965451a99689da72ad86fe361781c395300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:36:51 +0800 Subject: [PATCH 225/378] feat: +cache --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 50c414c97..6f4c3ec75 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -110,6 +110,9 @@ class BrainMemory(pydantic.BaseModel): async def set_history_summary(self, history_summary, redis_key, redis_conf): if self.historical_summary == history_summary: + if self.is_dirty: + await self.dumps(redis_key=redis_key, redis_conf=redis_conf) + self.is_dirty = False return self.historical_summary = history_summary From 63805c87f9c87de9b3823941a095b3f46b2f906b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:50:42 +0800 Subject: [PATCH 226/378] feat: +cache --- metagpt/memory/brain_memory.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 6f4c3ec75..a974d95f6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -31,9 +31,6 @@ class BrainMemory(pydantic.BaseModel): stack: List[Dict] = [] solution: List[Dict] = [] knowledge: List[Dict] = [] - # If the fingerprint of the history text is found in the `historical_summary_fingerprint`, - # it indicates that the text has already been incorporated into the `history summary`. - historical_summary_fingerprint: List[str] = [] historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False From 4dd9f7743f0d8dd3d4b2deb53b7a4d5e56d8bedc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 19:53:35 +0800 Subject: [PATCH 227/378] feat: +cache --- metagpt/memory/brain_memory.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a974d95f6..dedea3b41 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -5,8 +5,8 @@ @Author : mashenquan @File : brain_memory.py @Desc : Support memory for multiple tasks and multiple mainlines. +@Modified By: mashenquan, 2023/9/4. + redis memory cache. """ -import hashlib import json from enum import Enum from typing import Dict, List @@ -74,10 +74,6 @@ class BrainMemory(pydantic.BaseModel): return None return last_msg.content - @staticmethod - def get_md5(text: str) -> str: - return hashlib.md5(text.encode()).hexdigest() - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) From 7cb19c943c39279c3d811bb6525de0862e100e7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 21:21:46 +0800 Subject: [PATCH 228/378] fixbug: int --- metagpt/memory/brain_memory.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index dedea3b41..22af67236 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -115,8 +115,9 @@ class BrainMemory(pydantic.BaseModel): def add_history(self, msg: Message): if msg.id: - if int(msg.id) < int(self.last_history_id): + if self.to_int(msg.id, 0) < self.to_int(self.last_history_id, -1): return + self.last_history_id = str(self.to_int(msg.id, 0)) self.history.append(msg.dict()) self.is_dirty = True @@ -125,3 +126,10 @@ class BrainMemory(pydantic.BaseModel): if m.get("content") == text: return True return False + + @staticmethod + def to_int(v, default_value): + try: + return int(v) + except: + return default_value From 107ddbe308901713aa18d767c920e41e5e473e66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 22:38:58 +0800 Subject: [PATCH 229/378] refactor: talk prompt --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 83504b62d..a4cd78121 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -26,7 +26,7 @@ class TalkAction(Action): self._rsp = None @property - def prompt(self): + def prompt_old(self): prompt = "" if CONFIG.agent_description: prompt = ( @@ -46,7 +46,7 @@ class TalkAction(Action): return prompt @property - def formation_prompt(self): + def prompt(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 972337776de1d00f8997cdd73ab2c24df982cd96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 22:56:16 +0800 Subject: [PATCH 230/378] refactor: talk prompt --- metagpt/utils/redis.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/metagpt/utils/redis.py b/metagpt/utils/redis.py index b94eee8e2..48a18e7c9 100644 --- a/metagpt/utils/redis.py +++ b/metagpt/utils/redis.py @@ -4,6 +4,7 @@ # @Desc: { redis client } # @Date: 2022/11/28 10:12 import json +import traceback from datetime import timedelta from enum import Enum from typing import Awaitable, Callable, Dict, Optional, Union @@ -203,12 +204,19 @@ class Redis: async def get(self, key: str) -> str: if not self.is_valid() or not key: return None - v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key)) - return v + try: + v = await RedisManager.get_with_cache_info(redis_cache_info=RedisCacheInfo(key=key)) + return v + except Exception as e: + logger.exception(f"{e}, stack:{traceback.format_exc()}") + return None async def set(self, key: str, data: str, timeout_sec: int): if not self.is_valid() or not key: return - await RedisManager.set_with_cache_info( - redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data - ) + try: + await RedisManager.set_with_cache_info( + redis_cache_info=RedisCacheInfo(key=key, timeout=timeout_sec), value=data + ) + except Exception as e: + logger.exception(f"{e}, stack:{traceback.format_exc()}") From 557e82d8ef050466b3e465c17ccee695ff2d08ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:07:31 +0800 Subject: [PATCH 231/378] refactor: talk prompt --- metagpt/actions/talk_action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index a4cd78121..54c004602 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -57,6 +57,7 @@ class TalkAction(Action): prompt = TalkAction.__FORMATION_LOOSE__ for k, v in kvs.items(): prompt = prompt.replace(k, v) + logger.info(f"PROMPT: {prompt}") return prompt async def run(self, *args, **kwargs) -> ActionOutput: From 06c24c0eb4fc604ad1eff4980736e8ccc1b221ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:26:25 +0800 Subject: [PATCH 232/378] refactor: talk prompt --- metagpt/memory/brain_memory.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 22af67236..0c1ae024d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -117,7 +117,6 @@ class BrainMemory(pydantic.BaseModel): if msg.id: if self.to_int(msg.id, 0) < self.to_int(self.last_history_id, -1): return - self.last_history_id = str(self.to_int(msg.id, 0)) self.history.append(msg.dict()) self.is_dirty = True From d79a0638f2bc8f16b038327c39894add919669b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:34:54 +0800 Subject: [PATCH 233/378] fixbug: last_talk --- metagpt/memory/brain_memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 0c1ae024d..e2d9ad5ff 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -34,6 +34,7 @@ class BrainMemory(pydantic.BaseModel): historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False + last_talk: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) From b0966ca54133f9667a4ff173d17c5051b7993542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:38:43 +0800 Subject: [PATCH 234/378] fixbug: last_talk --- metagpt/memory/brain_memory.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e2d9ad5ff..60c563ed4 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -66,15 +66,6 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) - @property - def last_talk(self): - if len(self.history) == 0: - return None - last_msg = Message(**self.history[-1]) - if not last_msg.is_contain(MessageType.Talk.value): - return None - return last_msg.content - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) From 8075154a8db000c52f1db270b1907cb3f79d72f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Mon, 4 Sep 2023 23:46:12 +0800 Subject: [PATCH 235/378] fixbug: last_talk --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 9c80593f6..018a1fb01 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -125,7 +125,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary(history_text, max_words=500) + history_summary = await self._llm.get_summary(history_text, max_words=800) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) From 327e5fc9871cff1693fef512bd9a09645c69a7c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:02:21 +0800 Subject: [PATCH 236/378] fixbug: last_talk --- metagpt/provider/openai_api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index d0dd5b9d8..9f65dd905 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -226,15 +226,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def get_summary(self, text: str, max_words=200, keep_language: bool = False): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 + text_length = len(text) while max_count > 0: - if len(text) < max_token_count: + if text_length < max_token_count: return await self._get_summary(text=text, max_words=max_words, keep_language=keep_language) padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) + part_max_words = int(max_words / len(text_windows)) + 1 summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=max_words, keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From 1e7f0569183ba26b20e4c6060df0a047160d3e9c Mon Sep 17 00:00:00 2001 From: zhanglei Date: Tue, 5 Sep 2023 00:12:29 +0800 Subject: [PATCH 237/378] =?UTF-8?q?update:=20=E4=BC=98=E5=8C=96=E8=AE=B0?= =?UTF-8?q?=E5=BF=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/memory/brain_memory.py | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 22af67236..586285e4f 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -34,6 +34,7 @@ class BrainMemory(pydantic.BaseModel): historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False + last_talk: str = "" def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -65,15 +66,6 @@ class BrainMemory(pydantic.BaseModel): return "\n".join(texts) - @property - def last_talk(self): - if len(self.history) == 0: - return None - last_msg = Message(**self.history[-1]) - if not last_msg.is_contain(MessageType.Talk.value): - return None - return last_msg.content - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) From bc52a674e773416e6d4616ad2b2d13b6d27f404c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:20:34 +0800 Subject: [PATCH 238/378] fixbug: last_talk --- metagpt/provider/openai_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 9f65dd905..2539c5b70 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,10 +233,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = int(max_words / len(text_windows)) + 1 summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=200, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From dfc189510eb51928b732ebbcdfaa143a94252136 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:23:36 +0800 Subject: [PATCH 239/378] fixbug: last_talk --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2539c5b70..9406346ac 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,9 +233,10 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) + part_max_words = min(int(max_words / len(text_windows)) + 1, 200) summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=200, keep_language=keep_language) + response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: return summaries[0] From 8e1034afffcd3fbde4754ed64e49187f27beb672 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:27:37 +0800 Subject: [PATCH 240/378] fixbug: last_talk --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 9406346ac..157c353a8 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 200) + part_max_words = min(int(max_words / len(text_windows)) + 1, 100) summaries = [] for ws in text_windows: response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) From 998411a125e45a6265af7054081a2885e8a8d479 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:35:03 +0800 Subject: [PATCH 241/378] fixbug: last_talk --- metagpt/provider/openai_api.py | 2 +- metagpt/roles/assistant.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 157c353a8..2722491d0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 100) + part_max_words = min(int(max_words / len(text_windows)) + 1, 60) summaries = [] for ws in text_windows: response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 018a1fb01..4b2bfdab5 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -125,7 +125,7 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary(history_text, max_words=800) + history_summary = await self._llm.get_summary(history_text, max_words=800, keep_language=True) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) From 18a65470f031c65de06834c0651dd3574cda1c6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:37:32 +0800 Subject: [PATCH 242/378] fixbug: last_talk --- metagpt/memory/brain_memory.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 60c563ed4..92a71f69a 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -124,3 +124,8 @@ class BrainMemory(pydantic.BaseModel): return int(v) except: return default_value + + def pop_last_talk(self): + v = self.last_talk + self.last_talk = "" + return v From 22dbe3b224e8f7f0a8eedef942068deab4980ae0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:38:31 +0800 Subject: [PATCH 243/378] fixbug: last_talk --- metagpt/memory/brain_memory.py | 4 ++-- metagpt/roles/assistant.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 92a71f69a..2195da566 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -34,7 +34,7 @@ class BrainMemory(pydantic.BaseModel): historical_summary: str = "" last_history_id: str = "" is_dirty: bool = False - last_talk: str = "" + last_talk: str = None def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -127,5 +127,5 @@ class BrainMemory(pydantic.BaseModel): def pop_last_talk(self): v = self.last_talk - self.last_talk = "" + self.last_talk = None return v diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 4b2bfdab5..87127cbab 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -120,7 +120,7 @@ class Assistant(Role): async def refine_memory(self) -> str: history_text = self.memory.history_text - last_talk = self.memory.last_talk + last_talk = self.memory.pop_last_talk() if last_talk is None: # No user feedback, unsure if past conversation is finished. return None if history_text == "": From 9b2d6e492241493d3c5d4ef2c71152afc652acfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 00:41:47 +0800 Subject: [PATCH 244/378] fixbug: last_talk --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 2722491d0..bf2ca7f14 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -233,7 +233,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 60) + part_max_words = min(int(max_words / len(text_windows)) + 1, 100) summaries = [] for ws in text_windows: response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) @@ -243,6 +243,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # Merged and retry text = "\n".join(summaries) + text_length = len(text) max_count -= 1 # safeguard raise openai.error.InvalidRequestError("text too long") From 845cc8fbfd99626f1a6c740450382f0f3d49b2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 01:11:21 +0800 Subject: [PATCH 245/378] fixbug: last_talk --- metagpt/actions/talk_action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 54c004602..1c1a4e86d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -40,7 +40,7 @@ class TalkAction(Action): prompt += "According to the historical conversation above, " language = CONFIG.language or DEFAULT_LANGUAGE prompt += ( - f"Answer the following questions in {language}, and the answers must follow the Markdown format.\n " + f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) return prompt @@ -89,7 +89,7 @@ Statement: Your responses should align with the role-play agreement, maintaining [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions in {language}, and the answers must follow the Markdown format +Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" , "[ASK_END]" @@ -117,7 +117,7 @@ Statement: Your responses should maintaining the character's persona and habits. [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions in {language}, and the answers must follow the Markdown format +Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" , "[ASK_END]" From bcb6c7903e34c78baa9d2cb28a9555dea28ddfb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 01:48:09 +0800 Subject: [PATCH 246/378] fixbug: last_talk --- metagpt/actions/talk_action.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 1c1a4e86d..d6d18140a 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -90,8 +90,8 @@ Statement: Your responses should align with the role-play agreement, maintaining Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" - , "[ASK_END]" + , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" + , "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} @@ -118,8 +118,8 @@ Statement: Your responses should maintaining the character's persona and habits. Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]" - , "[ASK_END]" + , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" + , "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} From 5a6d5cc37dadb439a39bdccc3bfc20fac14414e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 10:41:25 +0800 Subject: [PATCH 247/378] fixbug: language professional --- metagpt/actions/talk_action.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index d6d18140a..cc30837b9 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -117,9 +117,9 @@ Statement: Your responses should maintaining the character's persona and habits. [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" - , "[ASK_BEGIN]", "[ASK_END]" in responses. +Statement: Unless you are a language professional, answer the following questions strictly in {language} +, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} From f54c507f06e6086720f163c2872c359a2ec28897 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 11:11:22 +0800 Subject: [PATCH 248/378] refactor: prompt --- metagpt/actions/talk_action.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index cc30837b9..ec151718e 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -74,6 +74,10 @@ class TalkAction(Action): "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. + “Personality” defines your language style。 + "Command" defines the action to do when command keyword is entered. + "Insight" provides a deeper understanding of the characters' inner traits. + "Initial" defines the initial setup of a character. Capacity and role: {role} Statement: Your responses should align with the role-play agreement, maintaining the @@ -89,9 +93,9 @@ Statement: Your responses should align with the role-play agreement, maintaining [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. -Statement: Answer the following questions strictly in {language}, and the answers must follow the Markdown format - , strictly excluding any tag likes "[HISTORY_BEGIN]", "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" - , "[ASK_BEGIN]", "[ASK_END]" in responses. +Statement: Unless you are a language professional, answer the following questions strictly in {language} +, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] {ask} @@ -103,6 +107,10 @@ Statement: Answer the following questions strictly in {language}, and the answer "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. + “Personality” defines your language style。 + "Command" defines the action to do when command keyword is entered. + "Insight" provides a deeper understanding of the characters' inner traits. + "Initial" defines the initial setup of a character. Capacity and role: {role} Statement: Your responses should maintaining the character's persona and habits. When faced with unrelated questions From c2c7f1c96d3494c4fd4cdb2f55e6922935077909 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 11:26:52 +0800 Subject: [PATCH 249/378] refactor: prompt --- metagpt/actions/talk_action.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index ec151718e..71ac5360a 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -98,7 +98,11 @@ Statement: Unless you are a language professional, answer the following question , "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] + + {ask} + + [ASK_END]""" __FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing; @@ -130,5 +134,9 @@ Statement: Unless you are a language professional, answer the following question , "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. [ASK_BEGIN] + + {ask} + + [ASK_END]""" From 54120e73562ebc8157eaf7f76a1890c958ed4fe3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 11:27:42 +0800 Subject: [PATCH 250/378] refactor: prompt --- metagpt/actions/talk_action.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 71ac5360a..c314b500d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -85,11 +85,15 @@ Statement: Your responses should align with the role-play agreement, maintaining your AI nature to preserve the character's image. [HISTORY_BEGIN] + {history} + [HISTORY_END] [KNOWLEDGE_BEGIN] + {knowledge} + [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. @@ -121,11 +125,15 @@ Statement: Your responses should maintaining the character's persona and habits. , playfully decline to answer without revealing your AI nature to preserve the character's image. [HISTORY_BEGIN] + {history} + [HISTORY_END] [KNOWLEDGE_BEGIN] + {knowledge} + [KNOWLEDGE_END] Statement: If the information is insufficient, you can search in the historical conversation or knowledge. From dec135ec833212400ea617d876f2c97ffff77916 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 12:26:36 +0800 Subject: [PATCH 251/378] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/document_store/faiss_store.py | 26 ++++++++-------- metagpt/memory/longterm_memory.py | 4 +-- metagpt/memory/memory_storage.py | 27 ++++++++--------- tests/metagpt/memory/test_longterm_memory.py | 32 ++++++++++---------- 4 files changed, 44 insertions(+), 45 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index fbfcb3086..16c152c1c 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -14,6 +14,7 @@ import faiss from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import FAISS +from metagpt.config import CONFIG from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore from metagpt.document_store.document import Document @@ -21,7 +22,7 @@ from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col='source', content_col='output'): + def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"): self.meta_col = meta_col self.content_col = content_col super().__init__(raw_data, cache_dir) @@ -37,11 +38,12 @@ class FaissStore(LocalStore): store.index = index return store - def _write(self, docs, metadatas, **kwargs): - store = FAISS.from_texts(docs, - OpenAIEmbeddings(openai_api_version="2020-11-07", - openai_api_key=kwargs.get("OPENAI_API_KEY")), - metadatas=metadatas) + def _write(self, docs, metadatas): + store = FAISS.from_texts( + docs, + OpenAIEmbeddings(openai_api_version="2020-11-07", openai_api_key=CONFIG.OPENAI_API_KEY), + metadatas=metadatas, + ) return store def persist(self): @@ -54,7 +56,7 @@ class FaissStore(LocalStore): pickle.dump(store, f) store.index = index - def search(self, query, expand_cols=False, sep='\n', *args, k=5, **kwargs): + def search(self, query, expand_cols=False, sep="\n", *args, k=5, **kwargs): rsp = self.store.similarity_search(query, k=k, **kwargs) logger.debug(rsp) if expand_cols: @@ -82,8 +84,8 @@ class FaissStore(LocalStore): raise NotImplementedError -if __name__ == '__main__': - faiss_store = FaissStore(DATA_PATH / 'qcs/qcs_4w.json') - logger.info(faiss_store.search('油皮洗面奶')) - faiss_store.add([f'油皮洗面奶-{i}' for i in range(3)]) - logger.info(faiss_store.search('油皮洗面奶')) +if __name__ == "__main__": + faiss_store = FaissStore(DATA_PATH / "qcs/qcs_4w.json") + logger.info(faiss_store.search("油皮洗面奶")) + faiss_store.add([f"油皮洗面奶-{i}" for i in range(3)]) + logger.info(faiss_store.search("油皮洗面奶")) diff --git a/metagpt/memory/longterm_memory.py b/metagpt/memory/longterm_memory.py index 041d335ac..df748037a 100644 --- a/metagpt/memory/longterm_memory.py +++ b/metagpt/memory/longterm_memory.py @@ -37,13 +37,13 @@ class LongTermMemory(Memory): self.add_batch(messages) self.msg_from_recover = False - def add(self, message: Message, **kwargs): + def add(self, message: Message): super(LongTermMemory, self).add(message) for action in self.rc.watch: if message.cause_by == action and not self.msg_from_recover: # currently, only add role's watching messages to its memory_storage # and ignore adding messages from recover repeatedly - self.memory_storage.add(message, **kwargs) + self.memory_storage.add(message) def remember(self, observed: list[Message], k=0) -> list[Message]: """ diff --git a/metagpt/memory/memory_storage.py b/metagpt/memory/memory_storage.py index 09cd67410..9afd524f0 100644 --- a/metagpt/memory/memory_storage.py +++ b/metagpt/memory/memory_storage.py @@ -5,16 +5,16 @@ @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -from typing import List from pathlib import Path +from typing import List from langchain.vectorstores.faiss import FAISS from metagpt.const import DATA_PATH, MEM_TTL +from metagpt.document_store.faiss_store import FaissStore from metagpt.logs import logger from metagpt.schema import Message -from metagpt.utils.serialize import serialize_message, deserialize_message -from metagpt.document_store.faiss_store import FaissStore +from metagpt.utils.serialize import deserialize_message, serialize_message class MemoryStorage(FaissStore): @@ -37,7 +37,7 @@ class MemoryStorage(FaissStore): def recover_memory(self, role_id: str) -> List[Message]: self.role_id = role_id - self.role_mem_path = Path(DATA_PATH / f'role_mem/{self.role_id}/') + self.role_mem_path = Path(DATA_PATH / f"role_mem/{self.role_id}/") self.role_mem_path.mkdir(parents=True, exist_ok=True) self.store = self._load() @@ -54,23 +54,23 @@ class MemoryStorage(FaissStore): def _get_index_and_store_fname(self): if not self.role_mem_path: - logger.error(f'You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory') + logger.error(f"You should call {self.__class__.__name__}.recover_memory fist when using LongTermMemory") return None, None - index_fpath = Path(self.role_mem_path / f'{self.role_id}.index') - storage_fpath = Path(self.role_mem_path / f'{self.role_id}.pkl') + index_fpath = Path(self.role_mem_path / f"{self.role_id}.index") + storage_fpath = Path(self.role_mem_path / f"{self.role_id}.pkl") return index_fpath, storage_fpath def persist(self): super(MemoryStorage, self).persist() - logger.debug(f'Agent {self.role_id} persist memory into local') + logger.debug(f"Agent {self.role_id} persist memory into local") - def add(self, message: Message, **kwargs) -> bool: - """ add message into memory storage""" + def add(self, message: Message) -> bool: + """add message into memory storage""" docs = [message.content] metadatas = [{"message_ser": serialize_message(message)}] if not self.store: # init Faiss - self.store = self._write(docs, metadatas, **kwargs) + self.store = self._write(docs, metadatas) self._initialized = True else: self.store.add_texts(texts=docs, metadatas=metadatas) @@ -82,10 +82,7 @@ class MemoryStorage(FaissStore): if not self.store: return [] - resp = self.store.similarity_search_with_score( - query=message.content, - k=k - ) + resp = self.store.similarity_search_with_score(query=message.content, k=k) # filter the result which score is smaller than the threshold filtered_resp = [] for item, score in resp: diff --git a/tests/metagpt/memory/test_longterm_memory.py b/tests/metagpt/memory/test_longterm_memory.py index 457e665fa..b77e9a955 100644 --- a/tests/metagpt/memory/test_longterm_memory.py +++ b/tests/metagpt/memory/test_longterm_memory.py @@ -4,11 +4,11 @@ @Desc : unittest of `metagpt/memory/longterm_memory.py` @Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ -from metagpt.config import Config -from metagpt.schema import Message from metagpt.actions import BossRequirement -from metagpt.roles.role import RoleContext +from metagpt.config import Config from metagpt.memory import LongTermMemory +from metagpt.roles.role import RoleContext +from metagpt.schema import Message def test_ltm_search(): @@ -17,28 +17,28 @@ def test_ltm_search(): openai_api_key = conf.openai_api_key assert len(openai_api_key) > 20 - role_id = 'UTUserLtm(Product Manager)' - rc = RoleContext(options=conf.runtime_options, watch=[BossRequirement]) + role_id = "UTUserLtm(Product Manager)" + rc = RoleContext(watch=[BossRequirement]) ltm = LongTermMemory() ltm.recover_memory(role_id, rc) - idea = 'Write a cli snake game' - message = Message(role='BOSS', content=idea, cause_by=BossRequirement) + idea = "Write a cli snake game" + message = Message(role="BOSS", content=idea, cause_by=BossRequirement) news = ltm.remember([message]) assert len(news) == 1 - ltm.add(message, **conf.runtime_options) + ltm.add(message) - sim_idea = 'Write a game of cli snake' - sim_message = Message(role='BOSS', content=sim_idea, cause_by=BossRequirement) + sim_idea = "Write a game of cli snake" + sim_message = Message(role="BOSS", content=sim_idea, cause_by=BossRequirement) news = ltm.remember([sim_message]) assert len(news) == 0 - ltm.add(sim_message, **conf.runtime_options) + ltm.add(sim_message) - new_idea = 'Write a 2048 web game' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a 2048 web game" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm.remember([new_message]) assert len(news) == 1 - ltm.add(new_message, **conf.runtime_options) + ltm.add(new_message) # restore from local index ltm_new = LongTermMemory() @@ -50,8 +50,8 @@ def test_ltm_search(): news = ltm_new.remember([sim_message]) assert len(news) == 0 - new_idea = 'Write a Battle City' - new_message = Message(role='BOSS', content=new_idea, cause_by=BossRequirement) + new_idea = "Write a Battle City" + new_message = Message(role="BOSS", content=new_idea, cause_by=BossRequirement) news = ltm_new.remember([new_message]) assert len(news) == 1 From 53030428c357ceda1ae11f830d850d9ea2e977d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 13:22:57 +0800 Subject: [PATCH 252/378] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/talk_action.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index c314b500d..e7b3d84c8 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -75,7 +75,6 @@ class TalkAction(Action): "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 - "Command" defines the action to do when command keyword is entered. "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. @@ -116,7 +115,6 @@ Statement: Unless you are a language professional, answer the following question "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 - "Command" defines the action to do when command keyword is entered. "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. From fa7e16192a76f1dc68374ae6f2767f2150b5a690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 13:28:28 +0800 Subject: [PATCH 253/378] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index e7b3d84c8..55e6e1aaa 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -72,7 +72,7 @@ class TalkAction(Action): "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; "Statement" defines the work detail you need to complete at this stage; - "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. @@ -112,7 +112,7 @@ Statement: Unless you are a language professional, answer the following question "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; "Statement" defines the work detail you need to complete at this stage; - "[ASK_BEGIN]" and [ASK_END] tags enclose the requirements for your to respond; + "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. “Personality” defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. From 6b59f28eb35ca7b975c3cfd4bbb38f900ea6bd51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 13:34:57 +0800 Subject: [PATCH 254/378] =?UTF-8?q?revert:=20=E6=94=B9=E7=94=A8CONFIG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 55e6e1aaa..6ec64d7f9 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -74,7 +74,7 @@ class TalkAction(Action): "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. - “Personality” defines your language style。 + "Personality" defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. @@ -114,7 +114,7 @@ Statement: Unless you are a language professional, answer the following question "Statement" defines the work detail you need to complete at this stage; "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. - “Personality” defines your language style。 + "Personality" defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. "Initial" defines the initial setup of a character. From 280fd62c94b8f19da3524dc398cdc879ae9e7456 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 14:08:00 +0800 Subject: [PATCH 255/378] revert: faiss store --- metagpt/document_store/faiss_store.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 16c152c1c..46b959d81 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -14,7 +14,6 @@ import faiss from langchain.embeddings import OpenAIEmbeddings from langchain.vectorstores import FAISS -from metagpt.config import CONFIG from metagpt.const import DATA_PATH from metagpt.document_store.base_store import LocalStore from metagpt.document_store.document import Document @@ -41,7 +40,7 @@ class FaissStore(LocalStore): def _write(self, docs, metadatas): store = FAISS.from_texts( docs, - OpenAIEmbeddings(openai_api_version="2020-11-07", openai_api_key=CONFIG.OPENAI_API_KEY), + OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas, ) return store From b9e3886e3012c8fe7f343d6bd165a861addfc43d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 14:08:29 +0800 Subject: [PATCH 256/378] revert: faiss store --- metagpt/document_store/faiss_store.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 46b959d81..55c07b920 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -38,11 +38,7 @@ class FaissStore(LocalStore): return store def _write(self, docs, metadatas): - store = FAISS.from_texts( - docs, - OpenAIEmbeddings(openai_api_version="2020-11-07"), - metadatas=metadatas, - ) + store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas) return store def persist(self): From e7ffd6dbc5ef4cef7036edff6178e2a6db27f450 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 14:11:24 +0800 Subject: [PATCH 257/378] revert: faiss store --- metagpt/document_store/faiss_store.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 55c07b920..7833bc706 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -4,7 +4,6 @@ @Time : 2023/5/25 10:20 @Author : alexanderwu @File : faiss_store.py -@Modified By: mashenquan, 2023/8/20. Remove global configuration `CONFIG`, enable configuration support for business isolation. """ import pickle from pathlib import Path From c204ee87071145ed7aa6214d635597eb0255d86d Mon Sep 17 00:00:00 2001 From: hongjiongteng Date: Tue, 5 Sep 2023 14:44:51 +0800 Subject: [PATCH 258/378] faiss init with kwargs --- metagpt/document_store/faiss_store.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/document_store/faiss_store.py b/metagpt/document_store/faiss_store.py index 7833bc706..be4748b50 100644 --- a/metagpt/document_store/faiss_store.py +++ b/metagpt/document_store/faiss_store.py @@ -20,9 +20,10 @@ from metagpt.logs import logger class FaissStore(LocalStore): - def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output"): + def __init__(self, raw_data: Path, cache_dir=None, meta_col="source", content_col="output", embedding_conf=None): self.meta_col = meta_col self.content_col = content_col + self.embedding_conf = embedding_conf or {} super().__init__(raw_data, cache_dir) def _load(self) -> Optional["FaissStore"]: @@ -37,7 +38,7 @@ class FaissStore(LocalStore): return store def _write(self, docs, metadatas): - store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07"), metadatas=metadatas) + store = FAISS.from_texts(docs, OpenAIEmbeddings(openai_api_version="2020-11-07", **self.embedding_conf), metadatas=metadatas) return store def persist(self): From 9779c578fad7c913b38ee97884af15f185f047a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 15:53:07 +0800 Subject: [PATCH 259/378] fixbug: prompt --- metagpt/actions/talk_action.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 6ec64d7f9..558145e0d 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -97,22 +97,17 @@ Statement: Your responses should align with the role-play agreement, maintaining Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Unless you are a language professional, answer the following questions strictly in {language} -, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" -, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. +, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses. -[ASK_BEGIN] - {ask} - - -[ASK_END]""" +""" __FORMATION_LOOSE__ = """Formation: "Capacity and role" defines the role you are currently playing; "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; "Statement" defines the work detail you need to complete at this stage; - "[ASK_BEGIN]" and [ASK_END] tags enclose the questions; "Constraint" defines the conditions that your responses must comply with. "Personality" defines your language style。 "Insight" provides a deeper understanding of the characters' inner traits. @@ -136,13 +131,9 @@ Statement: Your responses should maintaining the character's persona and habits. Statement: If the information is insufficient, you can search in the historical conversation or knowledge. Statement: Unless you are a language professional, answer the following questions strictly in {language} -, and the answers must follow the Markdown format, strictly excluding any tag likes "[HISTORY_BEGIN]" -, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]", "[ASK_BEGIN]", "[ASK_END]" in responses. - -[ASK_BEGIN] +, and the answers must follow the Markdown format. Strictly excluding any tag likes "[HISTORY_BEGIN]" +, "[HISTORY_END]", "[KNOWLEDGE_BEGIN]", "[KNOWLEDGE_END]" in responses. {ask} - - -[ASK_END]""" +""" From 40bbacd25d6af85e9a6810cd1333e05bc5818829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 16:08:31 +0800 Subject: [PATCH 260/378] revert: prompt --- metagpt/actions/talk_action.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 558145e0d..603736bc7 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -26,7 +26,7 @@ class TalkAction(Action): self._rsp = None @property - def prompt_old(self): + def prompt(self): prompt = "" if CONFIG.agent_description: prompt = ( @@ -34,10 +34,11 @@ class TalkAction(Action): f"maintaining the character's persona and habits. When faced with unrelated questions, playfully " f"decline to answer without revealing your AI nature to preserve the character's image.\n\n" ) - prompt += f"Background knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" + prompt += f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" - if self._history_summary != "": - prompt += "According to the historical conversation above, " + prompt += ( + "If the information is insufficient, you can search in the historical conversation or knowledge above." + ) language = CONFIG.language or DEFAULT_LANGUAGE prompt += ( f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " @@ -46,7 +47,7 @@ class TalkAction(Action): return prompt @property - def prompt(self): + def prompt_bad(self): kvs = { "{role}": CONFIG.agent_description or "", "{history}": self._history_summary or "", From 3f71ebb71ad01531de794f7b2caeefd3ad2ef942 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 16:13:28 +0800 Subject: [PATCH 261/378] revert: prompt --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 603736bc7..81caef013 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -37,7 +37,7 @@ class TalkAction(Action): prompt += f"Knowledge:\n{self._knowledge}\n\n" if self._knowledge else "" prompt += f"{self._history_summary}\n\n" prompt += ( - "If the information is insufficient, you can search in the historical conversation or knowledge above." + "If the information is insufficient, you can search in the historical conversation or knowledge above.\n" ) language = CONFIG.language or DEFAULT_LANGUAGE prompt += ( From 5c627df6c47fd8bd9257a4643a4fd0de49d7be82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 17:18:01 +0800 Subject: [PATCH 262/378] feat: +log --- metagpt/actions/talk_action.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 81caef013..4afed8014 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -44,6 +44,7 @@ class TalkAction(Action): f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) + logger.info(f"PROMPT: {prompt}") return prompt @property From c1aa93221086f094e3c661e3ac9f141f0f1b2168 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:10:16 +0800 Subject: [PATCH 263/378] feat: +iflytek tts --- .well-known/metagpt_oas3_api.yaml | 57 +++++++++++++++++++++++++++++++ metagpt/learn/text_to_speech.py | 29 ++++++++++++++-- requirements.txt | 3 +- 3 files changed, 85 insertions(+), 4 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 56c6f42d5..1e3cecb10 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -73,6 +73,63 @@ paths: '500': description: "Internal Server Error" + /tts/iflytek: + x-prerequisite: + - name: IFLYTEK_APP_ID + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + - name: IFLYTEK_API_KEY + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + - name: IFLYTEK_API_SECRET + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + post: + summary: "Convert Text to Base64-encoded .mp3 File Stream" + description: "For more details, check out: [iFlyTek](https://console.xfyun.cn/services/tts)" + operationId: iflytek_tts.oas3_iflytek_tts + requestBody: + required: true + content: + application/json: + schema: + type: object + required: + - text + properties: + text: + type: string + description: Text to convert + voice: + type: string + description: "Voice style, see: [iFlyTek Text-to_Speech](https://www.xfyun.cn/doc/tts/online_tts/API.html#%E6%8E%A5%E5%8F%A3%E8%B0%83%E7%94%A8%E6%B5%81%E7%A8%8B)" + default: "xiaoyan" + app_id: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + default: "" + api_key: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + default: "" + api_secret: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + default: "" + responses: + '200': + description: "Base64-encoded .mp3 file data if successful, otherwise an empty string." + content: + application/json: + schema: + type: object + properties: + wav_data: + type: string + format: base64 + '400': + description: "Bad Request" + '500': + description: "Internal Server Error" + + /txt2img/openai: x-prerequisite: - name: OPENAI_API_KEY diff --git a/metagpt/learn/text_to_speech.py b/metagpt/learn/text_to_speech.py index 81bc8512b..7c085c02f 100644 --- a/metagpt/learn/text_to_speech.py +++ b/metagpt/learn/text_to_speech.py @@ -11,6 +11,7 @@ import openai from metagpt.config import CONFIG from metagpt.const import BASE64_FORMAT from metagpt.tools.azure_tts import oas3_azsure_tts +from metagpt.tools.iflytek_tts import oas3_iflytek_tts from metagpt.utils.s3 import S3 @@ -22,6 +23,9 @@ async def text_to_speech( role="Girl", subscription_key="", region="", + iflytek_app_id="", + iflytek_api_key="", + iflytek_api_secret="", **kwargs, ): """Text to speech @@ -34,16 +38,35 @@ async def text_to_speech( :param text: The text used for voice conversion. :param subscription_key: key is used to access your Azure AI service API, see: `https://portal.azure.com/` > `Resource Management` > `Keys and Endpoint` :param region: This is the location (or region) of your resource. You may need to use this field when making calls to this API. - :return: Returns the Base64-encoded .wav file data if successful, otherwise an empty string. + :param iflytek_app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts` + :param iflytek_api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :param iflytek_api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :return: Returns the Base64-encoded .wav/.mp3 file data if successful, otherwise an empty string. """ - audio_declaration = "data:audio/wav;base64," + if (CONFIG.AZURE_TTS_SUBSCRIPTION_KEY and CONFIG.AZURE_TTS_REGION) or (subscription_key and region): + audio_declaration = "data:audio/wav;base64," base64_data = await oas3_azsure_tts(text, lang, voice, style, role, subscription_key, region) s3 = S3() url = await s3.cache(data=base64_data, file_ext=".wav", format=BASE64_FORMAT) if url: return f"[{text}]({url})" return audio_declaration + base64_data if base64_data else base64_data + if (CONFIG.IFLYTEK_APP_ID and CONFIG.IFLYTEK_API_KEY and CONFIG.IFLYTEK_API_SECRET) or ( + iflytek_app_id and iflytek_api_key and iflytek_api_secret + ): + audio_declaration = "data:audio/mp3;base64," + base64_data = await oas3_iflytek_tts( + text=text, app_id=iflytek_app_id, api_key=iflytek_api_key, api_secret=iflytek_api_secret + ) + s3 = S3() + url = await s3.cache(data=base64_data, file_ext=".mp3", format=BASE64_FORMAT) + if url: + return f"[{text}]({url})" + return audio_declaration + base64_data if base64_data else base64_data - raise openai.error.InvalidRequestError(message="AZURE_TTS_SUBSCRIPTION_KEY and AZURE_TTS_REGION error", param={}) + raise openai.error.InvalidRequestError( + message="AZURE_TTS_SUBSCRIPTION_KEY, AZURE_TTS_REGION, IFLYTEK_APP_ID, IFLYTEK_API_KEY, IFLYTEK_API_SECRET error", + param={}, + ) diff --git a/requirements.txt b/requirements.txt index 588b29e0b..2dd767026 100644 --- a/requirements.txt +++ b/requirements.txt @@ -42,4 +42,5 @@ connexion[swagger-ui] aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 aioboto3~=11.3.0 -redis==4.3.5 \ No newline at end of file +redis==4.3.5 +websocket-client \ No newline at end of file From f8aea281a85fde07459780f3e1f7e3b5a1e27e5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:11:22 +0800 Subject: [PATCH 264/378] feat: +iflytek tts --- metagpt/tools/iflytek_tts.py | 162 +++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 metagpt/tools/iflytek_tts.py diff --git a/metagpt/tools/iflytek_tts.py b/metagpt/tools/iflytek_tts.py new file mode 100644 index 000000000..a91d8091b --- /dev/null +++ b/metagpt/tools/iflytek_tts.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/8/17 +@Author : mashenquan +@File : iflytek_tts.py +@Desc : iFLYTEK TTS OAS3 api, which provides text-to-speech functionality +""" +import asyncio +import base64 +import hashlib +import hmac +import json +import uuid +from datetime import datetime +from enum import Enum +from pathlib import Path +from time import mktime +from typing import Optional +from urllib.parse import urlencode +from wsgiref.handlers import format_date_time + +import aiofiles +import websockets as websockets +from pydantic import BaseModel + +from metagpt.config import CONFIG +from metagpt.logs import logger + + +class IFlyTekTTSStatus(Enum): + STATUS_FIRST_FRAME = 0 # The first frame + STATUS_CONTINUE_FRAME = 1 # The intermediate frame + STATUS_LAST_FRAME = 2 # The last frame + + +class AudioData(BaseModel): + audio: str + status: int + ced: str + + +class IFlyTekTTSResponse(BaseModel): + code: int + message: str + data: Optional[AudioData] = None + sid: str + + +DEFAULT_IFLYTEK_VOICE = "xiaoyan" + + +class IFlyTekTTS(object): + def __init__(self, app_id: str, api_key: str, api_secret: str): + """ + :param app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts` + :param api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :param api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + """ + self.app_id = app_id or CONFIG.IFLYTEK_APP_ID + self.api_key = api_key or CONFIG.IFLYTEK_API_KEY + self.api_secret = api_secret or CONFIG.API_SECRET + + async def synthesize_speech(self, text, output_file: str, voice=DEFAULT_IFLYTEK_VOICE): + url = self._create_url() + data = { + "common": {"app_id": self.app_id}, + "business": {"aue": "lame", "sfl": 1, "auf": "audio/L16;rate=16000", "vcn": voice, "tte": "utf8"}, + "data": {"status": 2, "text": str(base64.b64encode(text.encode("utf-8")), "UTF8")}, + } + req = json.dumps(data) + async with websockets.connect(url) as websocket: + # send request + await websocket.send(req) + + # receive frames + async with aiofiles.open(str(output_file), "w") as writer: + while True: + v = await websocket.recv() + rsp = IFlyTekTTSResponse(**json.loads(v)) + if rsp.data: + await writer.write(rsp.data.audio) + if rsp.data.status != IFlyTekTTSStatus.STATUS_LAST_FRAME.value: + continue + break + + def _create_url(self): + """Create request url""" + url = "wss://tts-api.xfyun.cn/v2/tts" + # Generate a timestamp in RFC1123 format + now = datetime.now() + date = format_date_time(mktime(now.timetuple())) + + signature_origin = "host: " + "ws-api.xfyun.cn" + "\n" + signature_origin += "date: " + date + "\n" + signature_origin += "GET " + "/v2/tts " + "HTTP/1.1" + # Perform HMAC-SHA256 encryption + signature_sha = hmac.new( + self.api_secret.encode("utf-8"), signature_origin.encode("utf-8"), digestmod=hashlib.sha256 + ).digest() + signature_sha = base64.b64encode(signature_sha).decode(encoding="utf-8") + + authorization_origin = 'api_key="%s", algorithm="%s", headers="%s", signature="%s"' % ( + self.api_key, + "hmac-sha256", + "host date request-line", + signature_sha, + ) + authorization = base64.b64encode(authorization_origin.encode("utf-8")).decode(encoding="utf-8") + # Combine the authentication parameters of the request into a dictionary. + v = {"authorization": authorization, "date": date, "host": "ws-api.xfyun.cn"} + # Concatenate the authentication parameters to generate the URL. + url = url + "?" + urlencode(v) + return url + + +# Export +async def oas3_iflytek_tts(text: str, voice: str = "", app_id: str = "", api_key: str = "", api_secret: str = ""): + """Text to speech + For more details, check out:`https://www.xfyun.cn/doc/tts/online_tts/API.html` + + :param voice: Default `xiaoyan`. For more details, checkout: `https://www.xfyun.cn/doc/tts/online_tts/API.html#%E6%8E%A5%E5%8F%A3%E8%B0%83%E7%94%A8%E6%B5%81%E7%A8%8B` + :param text: The text used for voice conversion. + :param app_id: Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts` + :param api_key: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :param api_secret: WebAPI argument, see: `https://console.xfyun.cn/services/tts` + :return: Returns the Base64-encoded .mp3 file data if successful, otherwise an empty string. + + """ + if not app_id: + app_id = CONFIG.IFLYTEK_APP_ID + if not api_key: + api_key = CONFIG.IFLYTEK_API_KEY + if not api_secret: + api_secret = CONFIG.IFLYTEK_API_SECRET + if not voice: + voice = CONFIG.IFLYTEK_VOICE or DEFAULT_IFLYTEK_VOICE + + filename = Path(__file__).parent / (str(uuid.uuid4()).replace("-", "") + ".mp3") + try: + tts = IFlyTekTTS(app_id=app_id, api_key=api_key, api_secret=api_secret) + await tts.synthesize_speech(text=text, output_file=str(filename), voice=voice) + async with aiofiles.open(str(filename), mode="r") as reader: + base64_string = await reader.read() + except Exception as e: + logger.error(f"text:{text}, error:{e}") + base64_string = "" + finally: + filename.unlink() + + return base64_string + + +if __name__ == "__main__": + asyncio.get_event_loop().run_until_complete( + oas3_iflytek_tts( + text="你好，hello", + app_id="f7acef62", + api_key="fda72e3aa286042a492525816a5efa08", + api_secret="ZDk3NjdiMDBkODJlOWQ1NjRjMGI2NDY4", + ) + ) From 96aad1ce7745e7e39ae5dc82fbd2f59bf7ff144a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:25:50 +0800 Subject: [PATCH 265/378] feat: +log --- metagpt/tools/metagpt_oas3_api_svc.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/metagpt_oas3_api_svc.py b/metagpt/tools/metagpt_oas3_api_svc.py index 5c23f6566..2ff4c8225 100644 --- a/metagpt/tools/metagpt_oas3_api_svc.py +++ b/metagpt/tools/metagpt_oas3_api_svc.py @@ -7,8 +7,8 @@ @Desc : MetaGPT OpenAPI Specification 3.0 REST API service """ import asyncio -from pathlib import Path import sys +from pathlib import Path import connexion @@ -17,7 +17,7 @@ sys.path.append(str(Path(__file__).resolve().parent.parent.parent)) # fix-bug: def oas_http_svc(): """Start the OAS 3.0 OpenAPI HTTP service""" - app = connexion.AioHttpApp(__name__, specification_dir='../../.well-known/') + app = connexion.AioHttpApp(__name__, specification_dir="../../.well-known/") app.add_api("metagpt_oas3_api.yaml") app.add_api("openapi.yaml") app.run(port=8080) @@ -35,6 +35,7 @@ async def async_main(): def main(): + print("http://localhost:8080/oas3/ui/") oas_http_svc() From c800ad02d18bff6295af1a0d3a0fc1f50e9092a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:35:48 +0800 Subject: [PATCH 266/378] feat: +example --- .well-known/skills.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 009368dbe..d08d7aced 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -21,7 +21,9 @@ entities: - ask: 'A boy affectionate says "hello world"' answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' - ask: 'A boy says "你好"' - answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + answer: 'text_to_speech(text="你好", role="Boy", lang="Chinese")' + - ask: 'How to speak "你好"?' + answer: 'text_to_speech(text="你好", lang="Chinese")' returns: type: string format: base64 @@ -42,6 +44,10 @@ entities: answer: 'text_to_image(text="Draw a girl", size_type="512x512")' - ask: 'Draw an apple' answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + - ask: 'Draw an apple picture' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + - ask: 'Draw an apple image' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' returns: type: string format: base64 From f60b68f1c54bec7bd787e0620828887cc1a6ed09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:39:38 +0800 Subject: [PATCH 267/378] refactor: think prompt --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 87127cbab..a988572f4 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -62,7 +62,7 @@ class Assistant(Role): ) prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" - prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) From a71708addcdc19575b6ef7f5e36cbf871655867c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 19:56:41 +0800 Subject: [PATCH 268/378] feat: +ver --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2dd767026..3b2dc3106 100644 --- a/requirements.txt +++ b/requirements.txt @@ -43,4 +43,4 @@ aiohttp_jinja2 azure-cognitiveservices-speech==1.31.0 aioboto3~=11.3.0 redis==4.3.5 -websocket-client \ No newline at end of file +websocket-client==1.6.2 \ No newline at end of file From 50835b8c472b23238d351aadade7acf3b79e428d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:04:44 +0800 Subject: [PATCH 269/378] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index bf2ca7f14..06e06df69 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,7 +276,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"Paragraph 1:{text1}\n\nParagraph 2:{text2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From 246bf5ce00ab6a71fe8f97a297bbd44ed47a5bb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:07:55 +0800 Subject: [PATCH 270/378] refactor: think --- metagpt/provider/openai_api.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 06e06df69..68b0e4171 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,7 +276,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - command = f"Paragraph 1:{text1}\n\nParagraph 2:{text2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + p1 = text1.replace("\n", " ") + p2 = text2.replace("\n", " ") + command = f"Paragraph 1:{p1}\n\nParagraph 2:{p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From 0b412008c4e10626d124c2939dfcb9c43e529bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:11:51 +0800 Subject: [PATCH 271/378] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 68b0e4171..353ae46a0 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -278,7 +278,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): p1 = text1.replace("\n", " ") p2 = text2.replace("\n", " ") - command = f"Paragraph 1:{p1}\n\nParagraph 2:{p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"Paragraph 1: {p1}\n\nParagraph 2: {p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From caff43e1965acb87baf1011f54a9a77f68b4d041 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:24:46 +0800 Subject: [PATCH 272/378] refactor: think --- metagpt/provider/openai_api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 353ae46a0..fdf95f68c 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -280,14 +280,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): p2 = text2.replace("\n", " ") command = f"Paragraph 1: {p1}\n\nParagraph 2: {p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"RELATED:{rsp}\n\n{p1}\n{p2}") result, _ = self.extract_info(rsp) return result == "TRUE" async def rewrite(self, sentence: str, context: str): - command = ( - f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" - ) + command = f"{context}\n\nTaking into account the information above, please rephrase and provide the revised sentence:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:{rsp}\nFROM\n\n{sentence}") return rsp @staticmethod From b76ab1943656353eabde3320e7b8d4ffa1b24172 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 20:32:02 +0800 Subject: [PATCH 273/378] refactor: think --- metagpt/provider/openai_api.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index fdf95f68c..827a2e399 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -278,16 +278,14 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): p1 = text1.replace("\n", " ") p2 = text2.replace("\n", " ") - command = f"Paragraph 1: {p1}\n\nParagraph 2: {p2}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"Paragraph 1: {p2}\n\nParagraph 2: {p1}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"RELATED:{rsp}\n\n{p1}\n{p2}") result, _ = self.extract_info(rsp) return result == "TRUE" async def rewrite(self, sentence: str, context: str): command = f"{context}\n\nTaking into account the information above, please rephrase and provide the revised sentence:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"REWRITE:{rsp}\nFROM\n\n{sentence}") return rsp @staticmethod From 508fff69209ce0d34699bc4ac37dc13382f2b19e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:02:51 +0800 Subject: [PATCH 274/378] refactor: think --- metagpt/provider/openai_api.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 827a2e399..90fcd7ab3 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,15 +276,17 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - p1 = text1.replace("\n", " ") - p2 = text2.replace("\n", " ") - command = f"Paragraph 1: {p2}\n\nParagraph 2: {p1}\n\nIf the two Paragraphs above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" async def rewrite(self, sentence: str, context: str): - command = f"{context}\n\nTaking into account the information above, please rephrase and provide the revised sentence:\n{sentence}" + # command = ( + # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + # ) + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) return rsp From a9b56a6f56e1950e2b86f3c9c06f0c6f7bfed269 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:04:25 +0800 Subject: [PATCH 275/378] refactor: think --- metagpt/roles/assistant.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a988572f4..7fd1b1236 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -129,9 +129,10 @@ class Assistant(Role): await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) - if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. - last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) - return last_talk + # if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. + # last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + # return last_talk + last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) return last_talk From f450b61bc215ad70fbafb14e153a0cd905e203e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:13:10 +0800 Subject: [PATCH 276/378] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 90fcd7ab3..462d9d12d 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -286,7 +286,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # command = ( # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" # ) - command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text:\n{sentence}" + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) return rsp From cb17a17b4aa02552c6d99af6e18dbb8946ace33f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:20:39 +0800 Subject: [PATCH 277/378] refactor: think --- metagpt/roles/assistant.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 7fd1b1236..a988572f4 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -129,10 +129,9 @@ class Assistant(Role): await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) - # if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. - # last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) - # return last_talk - last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. + last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + return last_talk return last_talk From 1e39618b972bb9b9d55d53b1256c413451ecd289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:29:49 +0800 Subject: [PATCH 278/378] refactor: think --- metagpt/roles/assistant.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a988572f4..0a6237f42 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -60,8 +60,7 @@ class Assistant(Role): prompt += ( f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" ) - prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" - prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" + prompt += "If the user's intent is unclear, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon\n" prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) @@ -90,7 +89,6 @@ class Assistant(Role): skill, text = Assistant.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, - MessageType.Problem.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, } handler = handlers.get(skill, self.talk_handler) From 80b934d41ac5e6cdc559586fdfe5a699bad0c149 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:33:55 +0800 Subject: [PATCH 279/378] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 462d9d12d..7139c4946 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,7 +277,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE]." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From fa0b0b15114899e6724f081b7aa8f0dbfd9fbb6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:43:21 +0800 Subject: [PATCH 280/378] refactor: think --- metagpt/roles/assistant.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 0a6237f42..c0d1c3240 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -57,10 +57,7 @@ class Assistant(Role): prompt = f"Refer to this sentence:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += ( - f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - ) - prompt += "If the user's intent is unclear, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon\n" + prompt += f"If explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" logger.info(prompt) rsp = await self._llm.aask(prompt, []) From e2ffba863127b376afa53f3165816544206572bd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:51:14 +0800 Subject: [PATCH 281/378] refactor: think --- metagpt/roles/assistant.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index c0d1c3240..86a27cb18 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -58,7 +58,9 @@ class Assistant(Role): skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - prompt += "Otherwise, return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + prompt += ( + 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' + ) logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(rsp) From e25e19eb8fe6a4392766adf14f6456a649f023d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 22:59:36 +0800 Subject: [PATCH 282/378] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7139c4946..949b252b2 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,7 +277,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From 703b2a9a2418f3184ecad157b76e28112983cbec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:15:11 +0800 Subject: [PATCH 283/378] refactor: think --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 949b252b2..e352ff54f 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,7 +277,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE]:1 brief and clear. Otherwise, return [FALSE]:1 brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp) return result == "TRUE" From a147bdf92a306553bc93580b085102fb0efd7295 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:18:18 +0800 Subject: [PATCH 284/378] refactor: think --- metagpt/provider/openai_api.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index e352ff54f..bbceac1d2 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -318,8 +318,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return windows @staticmethod - def extract_info(input_string): - pattern = r"\[([A-Z]+)\]:\s*(.+)" + def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): match = re.match(pattern, input_string) if match: return match.group(1), match.group(2) From c8e24aa39b60cdea52664a29f6c52180c04d31be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:20:13 +0800 Subject: [PATCH 285/378] refactor: think --- metagpt/provider/openai_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index bbceac1d2..30b82b8dc 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -277,9 +277,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): async def is_related(self, text1, text2): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE]:1 brief and clear. Otherwise, return [FALSE]:1 brief and clear." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) - result, _ = self.extract_info(rsp) + result, _ = self.extract_info(rsp, pattern=r"\[([A-Z]+)\]\s*(.+)") return result == "TRUE" async def rewrite(self, sentence: str, context: str): From 558f80b238a1da513046351dab31c97598fa3282 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 5 Sep 2023 23:35:24 +0800 Subject: [PATCH 286/378] refactor: think --- metagpt/roles/assistant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 86a27cb18..428c1a70f 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,10 +54,10 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this sentence:\n {last_talk}\n" + prompt = f"Refer to this text:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += f"If explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + prompt += f"If the text explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += ( 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' ) From 04231088c7717241df1da275f1c553854188897c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:14:31 +0800 Subject: [PATCH 287/378] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 428c1a70f..6530a3cac 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -57,7 +57,7 @@ class Assistant(Role): prompt = f"Refer to this text:\n {last_talk}\n" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += f"If the text explicitly want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += ( 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' ) From ac211ae3a6ed9df419585b70d6a6765223a6aaf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:17:21 +0800 Subject: [PATCH 288/378] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 6530a3cac..516f78b0e 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -63,7 +63,7 @@ class Assistant(Role): ) logger.info(prompt) rsp = await self._llm.aask(prompt, []) - logger.info(rsp) + logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: From 092243670f7e9e716187be27843c7d11aff6b832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:23:29 +0800 Subject: [PATCH 289/378] feat: +log --- metagpt/provider/openai_api.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 30b82b8dc..99f281964 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -280,6 +280,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) result, _ = self.extract_info(rsp, pattern=r"\[([A-Z]+)\]\s*(.+)") + p2 = text2.replace("\n", "") + p1 = text1.replace("\n", "") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") return result == "TRUE" async def rewrite(self, sentence: str, context: str): @@ -288,6 +291,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # ) command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}") return rsp @staticmethod From 6f55709ec599f804dcaefd86b4260e6ec6024f5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:31:21 +0800 Subject: [PATCH 290/378] feat: +log --- metagpt/provider/openai_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 99f281964..d84109f6a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -279,11 +279,11 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) - result, _ = self.extract_info(rsp, pattern=r"\[([A-Z]+)\]\s*(.+)") + result = True if "TRUE" in rsp else False p2 = text2.replace("\n", "") p1 = text1.replace("\n", "") logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") - return result == "TRUE" + return result async def rewrite(self, sentence: str, context: str): # command = ( From 8f8a5e185a84ebccc5bad58fa0a21fa963613cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:41:30 +0800 Subject: [PATCH 291/378] refactor: think --- metagpt/roles/assistant.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 516f78b0e..bae1b6c79 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,13 +54,12 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this text:\n {last_talk}\n" + prompt = "" skills = self.skills.get_skill_list() for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - prompt += ( - 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx' - ) + prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' + prompt = f"Now the text is: {last_talk}" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 8695a042e99cc61efeedbf2bde9c2db0525f5751 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 10:49:06 +0800 Subject: [PATCH 292/378] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index bae1b6c79..a615c3933 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt = f"Now the text is: {last_talk}" + prompt += f"Now the text is: {last_talk}" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 2ff563e6b6261bf04116991f92ba0c3bacad920d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:01:21 +0800 Subject: [PATCH 293/378] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index a615c3933..743ec7c43 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now the text is: {last_talk}" + prompt += f"Now determine the appropriate pattern for the text: {last_talk}" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 5f5fda42730cbc2e6441e20260bf246a6ee98e10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:03:25 +0800 Subject: [PATCH 294/378] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 743ec7c43..07991da1a 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now determine the appropriate pattern for the text: {last_talk}" + prompt += f"Now determine the appropriate pattern for the text: {last_talk}\n" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From db72848965b29400a9235f7432a0a85cd3206ba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:17:32 +0800 Subject: [PATCH 295/378] refactor: think --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 07991da1a..d310fca7c 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,7 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now determine the appropriate pattern for the text: {last_talk}\n" + prompt += f"Now what specific action does the text explicitly ask for: {last_talk}\n" logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") From 341bbbe4ba8a1e959158724196b9a8529d4211dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 11:33:40 +0800 Subject: [PATCH 296/378] refactor: think --- metagpt/roles/assistant.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index d310fca7c..bef2cf53c 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -59,8 +59,7 @@ class Assistant(Role): for desc, name in skills.items(): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' - prompt += f"Now what specific action does the text explicitly ask for: {last_talk}\n" - logger.info(prompt) + prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n" rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) From 03019a304bfd342b2a0d5ed62b5a262bb513e8e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 12:13:13 +0800 Subject: [PATCH 297/378] refactor: think --- metagpt/actions/talk_action.py | 1 + metagpt/provider/openai_api.py | 21 +++++++++++++-------- metagpt/roles/assistant.py | 14 +++++--------- 3 files changed, 19 insertions(+), 17 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 81caef013..4afed8014 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -44,6 +44,7 @@ class TalkAction(Action): f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) + logger.info(f"PROMPT: {prompt}") return prompt @property diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index bf2ca7f14..d84109f6a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -276,16 +276,22 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return response async def is_related(self, text1, text2): - command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await self.aask(msg=command, system_msgs=[]) - result, _ = self.extract_info(rsp) - return result == "TRUE" + result = True if "TRUE" in rsp else False + p2 = text2.replace("\n", "") + p1 = text1.replace("\n", "") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") + return result async def rewrite(self, sentence: str, context: str): - command = ( - f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" - ) + # command = ( + # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + # ) + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}") return rsp @staticmethod @@ -316,8 +322,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return windows @staticmethod - def extract_info(input_string): - pattern = r"\[([A-Z]+)\]:\s*(.+)" + def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): match = re.match(pattern, input_string) if match: return match.group(1), match.group(2) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 87127cbab..ac80a4bc8 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -54,18 +54,15 @@ class Assistant(Role): last_talk = await self.refine_memory() if not last_talk: return False - prompt = f"Refer to this sentence:\n {last_talk}\n" + prompt = "" skills = self.skills.get_skill_list() for desc, name in skills.items(): - prompt += ( - f"If want you to do {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" - ) - prompt += "If the preceding text presents a complete question and solution, rewrite and return `[SOLUTION]: {problem}` brief and clear. For instance: [SOLUTION]: Solution for distributing watermelon\n" - prompt += "If the preceding text presents an unresolved issue and its corresponding discussion, rewrite and return `[PROBLEM]: {problem}` brief and clear. For instance: [PROBLEM]: How to distribute watermelon?\n" - prompt += "Otherwise, rewrite and return `[TALK]: {talk}` brief and clear. For instance: [TALK]: distribute watermelon" + prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" + prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' + prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n" logger.info(prompt) rsp = await self._llm.aask(prompt, []) - logger.info(rsp) + logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) async def act(self) -> ActionOutput: @@ -90,7 +87,6 @@ class Assistant(Role): skill, text = Assistant.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, - MessageType.Problem.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, } handler = handlers.get(skill, self.talk_handler) From 4e0b2898a6a54993738448991699e81dd58bd577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 12:24:20 +0800 Subject: [PATCH 298/378] refactor: think --- metagpt/roles/assistant.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index ac80a4bc8..bef2cf53c 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -60,7 +60,6 @@ class Assistant(Role): prompt += f"If the text explicitly want you to {desc}, return `[SKILL]: {name}` brief and clear. For instance: [SKILL]: {name}\n" prompt += 'Otherwise, return `[TALK]: {talk}` brief and clear. For instance: if {talk} is "xxxx" return [TALK]: xxxx\n\n' prompt += f"Now what specific action is explicitly mentioned in the text: {last_talk}\n" - logger.info(prompt) rsp = await self._llm.aask(prompt, []) logger.info(f"THINK: {prompt}\n, THINK RESULT: {rsp}\n") return await self._plan(rsp, last_talk=last_talk) From c792cf09ecb642faf9bc628edf920b43847f83f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 13:18:03 +0800 Subject: [PATCH 299/378] refactor: think --- metagpt/roles/assistant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index bef2cf53c..cd1932f82 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -93,6 +93,7 @@ class Assistant(Role): async def talk_handler(self, text, **kwargs) -> bool: history = self.memory.history_text + text = kwargs.get("last_talk") or text action = TalkAction( talk=text, knowledge=self.memory.get_knowledge(), history_summary=history, llm=self._llm, **kwargs ) From eec0fbde6d0a4563b166b5cab929a65bc70c518b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 15:02:36 +0800 Subject: [PATCH 300/378] refactor: disable log --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 2195da566..f309b532e 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -72,7 +72,7 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return BrainMemory() v = await redis.get(key=redis_key) - logger.info(f"REDIS GET {redis_key} {v}") + logger.debug(f"REDIS GET {redis_key} {v}") if v: data = json.loads(v) bm = BrainMemory(**data) @@ -86,7 +86,7 @@ class BrainMemory(pydantic.BaseModel): return False v = self.json() await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) - logger.info(f"REDIS SET {redis_key} {v}") + logger.debug(f"REDIS SET {redis_key} {v}") self.is_dirty = False @staticmethod From b3be30bdad534836d1bdaa168ae2a8a9d9e42245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 6 Sep 2023 15:12:41 +0800 Subject: [PATCH 301/378] refactor: log --- metagpt/actions/skill_action.py | 3 +-- metagpt/actions/talk_action.py | 5 ++--- metagpt/provider/openai_api.py | 12 ++++++------ 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/metagpt/actions/skill_action.py b/metagpt/actions/skill_action.py index 758591fdd..f629cfcbf 100644 --- a/metagpt/actions/skill_action.py +++ b/metagpt/actions/skill_action.py @@ -40,9 +40,8 @@ class ArgumentsParingAction(Action): async def run(self, *args, **kwargs) -> ActionOutput: prompt = self.prompt - logger.info(prompt) rsp = await self.llm.aask(msg=prompt, system_msgs=[]) - logger.info(rsp) + logger.debug(f"SKILL:{prompt}\n, RESULT:{rsp}") self.args = ArgumentsParingAction.parse_arguments(skill_name=self.skill.name, txt=rsp) self.rsp = ActionOutput(content=rsp) return self.rsp diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 4afed8014..0e3762798 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -44,7 +44,7 @@ class TalkAction(Action): f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.\n " f"{self._talk}" ) - logger.info(f"PROMPT: {prompt}") + logger.debug(f"PROMPT: {prompt}") return prompt @property @@ -64,9 +64,8 @@ class TalkAction(Action): async def run(self, *args, **kwargs) -> ActionOutput: prompt = self.prompt - logger.info(prompt) rsp = await self.llm.aask(msg=prompt, system_msgs=[]) - logger.info(rsp) + logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n") self._rsp = ActionOutput(content=rsp) return self._rsp diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index d84109f6a..863475f52 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -257,9 +257,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): else: command = f"Translate the above content into a summary of less than {max_words} words." msg = text + "\n\n" + command - logger.info(f"summary ask:{msg}") + logger.debug(f"summary ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) - logger.info(f"summary rsp: {response}") + logger.debug(f"summary rsp: {response}") return response async def get_context_title(self, text: str, max_words=5) -> str: @@ -270,9 +270,9 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): command = f"Translate the above summary into a {language} title of less than {max_words} words." summaries = [summary, command] msg = "\n".join(summaries) - logger.info(f"title ask:{msg}") + logger.debug(f"title ask:{msg}") response = await self.aask(msg=msg, system_msgs=[]) - logger.info(f"title rsp: {response}") + logger.debug(f"title rsp: {response}") return response async def is_related(self, text1, text2): @@ -282,7 +282,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): result = True if "TRUE" in rsp else False p2 = text2.replace("\n", "") p1 = text1.replace("\n", "") - logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n") return result async def rewrite(self, sentence: str, context: str): @@ -291,7 +291,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): # ) command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}") + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n") return rsp @staticmethod From 832294809b097793dff3472b1183aed37f8f5c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:01:27 +0800 Subject: [PATCH 302/378] feat: + LLMType --- metagpt/llm.py | 24 ++++++++++++++++++++++-- metagpt/provider/__init__.py | 4 +++- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 6a9a9132f..0ef23d0be 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -4,17 +4,37 @@ @Time : 2023/5/11 14:45 @Author : alexanderwu @File : llm.py +@Modified By: mashenquan, 2023 """ +from enum import Enum from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.openai_api import OpenAIGPTAPI as LLM + +class LLMType(Enum): + OPENAI = "OpenAI" + METAGPT = "MetaGPT" + UNKNOWN = "UNKNOWN" + + @classmethod + def get(cls, value): + for member in cls: + if member.value == value: + return member + return cls.UNKNOWN + + @property + def UNKNOWN(self): + return LLMType.UNKNOWN + + DEFAULT_LLM = LLM() CLAUDE_LLM = Claude() async def ai_func(prompt): """使用LLM进行QA - QA with LLMs - """ + QA with LLMs + """ return await DEFAULT_LLM.aask(prompt) diff --git a/metagpt/provider/__init__.py b/metagpt/provider/__init__.py index 56dc19b4b..9895aa7fc 100644 --- a/metagpt/provider/__init__.py +++ b/metagpt/provider/__init__.py @@ -4,9 +4,11 @@ @Time : 2023/5/5 22:59 @Author : alexanderwu @File : __init__.py +@Modified By: mashenquan, 2023/9/8. Add `MetaGPTLLMAPI` """ from metagpt.provider.openai_api import OpenAIGPTAPI +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI -__all__ = ["OpenAIGPTAPI"] +__all__ = ["OpenAIGPTAPI", "MetaGPTLLMAPI"] From 154f67c5e32467a9a21b5cb979aed25fa7e32520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:06:39 +0800 Subject: [PATCH 303/378] feat: + LLMType --- metagpt/llm.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 0ef23d0be..e31eee908 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -9,7 +9,8 @@ from enum import Enum from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.openai_api import OpenAIGPTAPI as LLM +from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM +from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM class LLMType(Enum): @@ -29,7 +30,8 @@ class LLMType(Enum): return LLMType.UNKNOWN -DEFAULT_LLM = LLM() +DEFAULT_LLM = OpenAI_LLM() +DEFAULT_METAGPT_LLM = MetaGPT_LLM() CLAUDE_LLM = Claude() From e316fe4d60ac8b2de96729fdc998d0db6069d1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:20:27 +0800 Subject: [PATCH 304/378] feat: + LLMType --- metagpt/llm.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/metagpt/llm.py b/metagpt/llm.py index e31eee908..87ce8fa5b 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,6 +8,7 @@ """ from enum import Enum +from metagpt.config import CONFIG from metagpt.provider.anthropic_api import Claude2 as Claude from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM @@ -40,3 +41,10 @@ async def ai_func(prompt): QA with LLMs """ return await DEFAULT_LLM.aask(prompt) + + +class LLMFactory: + @staticmethod + async def new_llm() -> object: + llm = OpenAI_LLM() if CONFIG.LLM_TYPE == LLMType.OPENAI.value else MetaGPT_LLM() + return llm From c513712928e58ae3782819b29accf515ff366de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:43:34 +0800 Subject: [PATCH 305/378] feat: + kwargs --- metagpt/provider/openai_api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 863475f52..64267975e 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -223,7 +223,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=200, keep_language: bool = False): + async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 text_length = len(text) @@ -262,7 +262,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): logger.debug(f"summary rsp: {response}") return response - async def get_context_title(self, text: str, max_words=5) -> str: + async def get_context_title(self, text: str, max_words=5, **kwargs) -> str: """Generate text title""" summary = await self.get_summary(text, max_words=500) From a41fe2494e34af249ebdd530f0a8cecb2b3a259c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 11:55:36 +0800 Subject: [PATCH 306/378] feat: +LLMType --- metagpt/llm.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 87ce8fa5b..93cbcaaf6 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -26,10 +26,6 @@ class LLMType(Enum): return member return cls.UNKNOWN - @property - def UNKNOWN(self): - return LLMType.UNKNOWN - DEFAULT_LLM = OpenAI_LLM() DEFAULT_METAGPT_LLM = MetaGPT_LLM() From 3a4f31b51787f2e60bed4efda45f63d49e1637ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 12:00:30 +0800 Subject: [PATCH 307/378] feat: +LLMType --- metagpt/actions/action.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index e4b9613ad..c52caaa40 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -5,6 +5,7 @@ @Author : alexanderwu @File : action.py @Modified By: mashenquan, 2023/8/20. Add function return annotations. +@Modified By: mashenquan, 2023/9/8. Replace LLM with LLMFactory """ from __future__ import annotations @@ -14,16 +15,17 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.llm import LLM +from metagpt.llm import LLMFactory from metagpt.logs import logger +from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.common import OutputParser class Action(ABC): - def __init__(self, name: str = "", context=None, llm: LLM = None): + def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None): self.name: str = name if llm is None: - llm = LLM() + llm = LLMFactory.new_llm() self.llm = llm self.context = context self.prefix = "" From c7bc975cf20c926dcc52756efcc3038c9b6b30f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 12:02:43 +0800 Subject: [PATCH 308/378] fixbug: LLM() --- metagpt/roles/role.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/roles/role.py b/metagpt/roles/role.py index b1ace19fa..6d774b0b4 100644 --- a/metagpt/roles/role.py +++ b/metagpt/roles/role.py @@ -16,7 +16,7 @@ from pydantic import BaseModel, Field from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import OPTIONS -from metagpt.llm import LLM +from metagpt.llm import LLMFactory from metagpt.logs import logger from metagpt.memory import LongTermMemory, Memory from metagpt.schema import Message, MessageTag @@ -113,7 +113,7 @@ class Role: constraints = Role.format_value(constraints) desc = Role.format_value(desc) - self._llm = LLM() + self._llm = LLMFactory.new_llm() self._setting = RoleSetting(name=name, profile=profile, goal=goal, constraints=constraints, desc=desc) self._states = [] self._actions = [] From 2324c1c6dcc334cfe368b3e4252db3060dbfbba4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 12:44:49 +0800 Subject: [PATCH 309/378] fixbug: LLM() --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 93cbcaaf6..4772d2e6e 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -41,6 +41,6 @@ async def ai_func(prompt): class LLMFactory: @staticmethod - async def new_llm() -> object: + def new_llm() -> object: llm = OpenAI_LLM() if CONFIG.LLM_TYPE == LLMType.OPENAI.value else MetaGPT_LLM() return llm From ef485e7400546e5577f3ca59fcc089c811e13f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 13:52:36 +0800 Subject: [PATCH 310/378] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 21 +++++++++++++++++++++ metagpt/roles/assistant.py | 4 +++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index c27e7132d..f8c4ac1ed 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -5,10 +5,13 @@ @File : metagpt_llm_api.py @Desc : MetaGPT LLM related APIs """ +import json import openai +from pydantic import BaseModel from metagpt.config import CONFIG +from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI from metagpt.provider.openai_api import RateLimiter @@ -31,3 +34,21 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): openai.api_type = CONFIG.METAGPT_API_TYPE openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 + + async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): + summary = [] + + class QuweryAnswerPair(BaseModel): + ask: str + answer: str + + rh = reversed(memory.history) + ix = 0 + while ix < len(rh): + t = rh[ix] + print(t) + # 如果 t是ask, continue + pass + + data = json.dumps(summary) + return data diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index cd1932f82..0a796ac11 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -121,7 +121,9 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary(history_text, max_words=800, keep_language=True) + history_summary = await self._llm.get_summary( + history_text, max_words=800, keep_language=True, memory=self.memory + ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS ) From 6848d189cfd5c5d5df05d53fb825c64a85121090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:15:03 +0800 Subject: [PATCH 311/378] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index f8c4ac1ed..0688e1878 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -27,12 +27,12 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): RateLimiter.__init__(self, rpm=self.rpm) def __init_openai(self, *args, **kwargs): - openai.api_key = CONFIG.METAGPT_API_KEY - if CONFIG.METAGPT_API_BASE: - openai.api_base = CONFIG.METAGPT_API_BASE - if CONFIG.METAGPT_API_TYPE: - openai.api_type = CONFIG.METAGPT_API_TYPE - openai.api_version = CONFIG.METAGPT_API_VERSION + # openai.api_key = CONFIG.METAGPT_API_KEY + # if CONFIG.METAGPT_API_BASE: + # openai.api_base = CONFIG.METAGPT_API_BASE + # if CONFIG.METAGPT_API_TYPE: + # openai.api_type = CONFIG.METAGPT_API_TYPE + # openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): From 451b3510552fc22eef69b4e6f44e0a4caea7f75a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:18:19 +0800 Subject: [PATCH 312/378] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 0688e1878..17c2b3ab8 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -20,20 +20,11 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.__init_openai() self.llm = openai self.model = CONFIG.METAGPT_API_MODEL self.auto_max_tokens = False - RateLimiter.__init__(self, rpm=self.rpm) - - def __init_openai(self, *args, **kwargs): - # openai.api_key = CONFIG.METAGPT_API_KEY - # if CONFIG.METAGPT_API_BASE: - # openai.api_base = CONFIG.METAGPT_API_BASE - # if CONFIG.METAGPT_API_TYPE: - # openai.api_type = CONFIG.METAGPT_API_TYPE - # openai.api_version = CONFIG.METAGPT_API_VERSION self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 + RateLimiter.__init__(self, rpm=self.rpm) async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): summary = [] From 098027d249e709c4a939d8feb042e76f26ef0116 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:23:36 +0800 Subject: [PATCH 313/378] feat: + summary --- metagpt/const.py | 6 ------ metagpt/provider/metagpt_llm_api.py | 9 +-------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/metagpt/const.py b/metagpt/const.py index e9fa118d7..2323e3b6d 100644 --- a/metagpt/const.py +++ b/metagpt/const.py @@ -48,12 +48,6 @@ BRAIN_MEMORY = "BRAIN_MEMORY" SKILL_PATH = "SKILL_PATH" SERPER_API_KEY = "SERPER_API_KEY" -# Key Definitions for MetaGPT LLM -METAGPT_API_MODEL = "METAGPT_API_MODEL" -METAGPT_API_KEY = "METAGPT_API_KEY" -METAGPT_API_BASE = "METAGPT_API_BASE" -METAGPT_API_TYPE = "METAGPT_API_TYPE" -METAGPT_API_VERSION = "METAGPT_API_VERSION" # format BASE64_FORMAT = "base64" diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 17c2b3ab8..c21ffd650 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -7,24 +7,17 @@ """ import json -import openai from pydantic import BaseModel -from metagpt.config import CONFIG from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI -from metagpt.provider.openai_api import RateLimiter class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): - self.llm = openai - self.model = CONFIG.METAGPT_API_MODEL - self.auto_max_tokens = False - self.rpm = int(CONFIG.RPM) if CONFIG.RPM else 10 - RateLimiter.__init__(self, rpm=self.rpm) + super().__init__() async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): summary = [] From 239f68d40d1c49b94736344a94e8459fee43c535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:30:36 +0800 Subject: [PATCH 314/378] feat: + summary --- metagpt/actions/action.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/actions/action.py b/metagpt/actions/action.py index c52caaa40..92608f448 100644 --- a/metagpt/actions/action.py +++ b/metagpt/actions/action.py @@ -15,7 +15,6 @@ from typing import Optional from tenacity import retry, stop_after_attempt, wait_fixed from metagpt.actions.action_output import ActionOutput -from metagpt.llm import LLMFactory from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.common import OutputParser @@ -25,6 +24,8 @@ class Action(ABC): def __init__(self, name: str = "", context=None, llm: BaseGPTAPI = None): self.name: str = name if llm is None: + from metagpt.llm import LLMFactory + llm = LLMFactory.new_llm() self.llm = llm self.context = context From bda4132a9062a995616b1fd6ac93d35d218812dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 14:59:40 +0800 Subject: [PATCH 315/378] feat: + summary --- metagpt/schema.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/schema.py b/metagpt/schema.py index 8f8e4030f..9bf85174b 100644 --- a/metagpt/schema.py +++ b/metagpt/schema.py @@ -76,6 +76,7 @@ class Message: "sent_from": self.sent_from, "send_to": self.send_to, "tags": self.tags, + "id": self.id, } m = {"content": self.content} From 7723df1455b2a646e431d721ca9326d7872bb67a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 15:55:50 +0800 Subject: [PATCH 316/378] feat: + summary --- metagpt/provider/metagpt_llm_api.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index c21ffd650..06476f63b 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -6,10 +6,10 @@ @Desc : MetaGPT LLM related APIs """ import json +from typing import Dict, List from pydantic import BaseModel -from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI @@ -19,17 +19,22 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init__(self): super().__init__() - async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs): + async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs): summary = [] + class HisMsg(BaseModel): + content: str + tags: set + id: str + class QuweryAnswerPair(BaseModel): ask: str answer: str - rh = reversed(memory.history) + rh = reversed(history) ix = 0 while ix < len(rh): - t = rh[ix] + t = HisMsg(**rh[ix]) print(t) # 如果 t是ask, continue pass From a0ad7872f7dfc946ddc27d5e106fc3ab82130dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 15:57:24 +0800 Subject: [PATCH 317/378] feat: + summary --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 0a796ac11..5d04c2d6f 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -122,7 +122,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary( - history_text, max_words=800, keep_language=True, memory=self.memory + history_text, max_words=800, keep_language=True, history=self.memory.history ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS From 8a0644a496fb956106dbcbc5697cd48617b85009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 15:58:13 +0800 Subject: [PATCH 318/378] feat: + summary --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 5d04c2d6f..2f9059210 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -122,7 +122,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary( - history_text, max_words=800, keep_language=True, history=self.memory.history + text=history_text, max_words=800, keep_language=True, history=self.memory.history ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS From 4c9a5d8dda1238dbe0056243d3ef1860f6be0d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 16:45:01 +0800 Subject: [PATCH 319/378] feat: truncated history --- metagpt/provider/metagpt_llm_api.py | 53 ++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 06476f63b..d8d06aeaa 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -10,34 +10,55 @@ from typing import Dict, List from pydantic import BaseModel +from metagpt.memory.brain_memory import MessageType from metagpt.provider import OpenAIGPTAPI +class HisMsg(BaseModel): + content: str + tags: set + id: str + + +class Conversion(BaseModel): + """See: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb""" + + role: str + content: str + + class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): super().__init__() - async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs): + async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs) -> str: + """ + Return string in the following format： + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Knock knock."}, + {"role": "assistant", "content": "Who's there?"}, + {"role": "user", "content": "Orange."}, + ] + """ summary = [] - class HisMsg(BaseModel): - content: str - tags: set - id: str + total_length = 0 + for m in reversed(history): + msg = HisMsg(**m) + c = Conversion(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) + length_delta = len(msg.content) + if total_length + length_delta > max_words: + left = max_words - total_length + if left > 0: + c.content = msg.content[0:left] + summary.insert(0, c.dict()) + break - class QuweryAnswerPair(BaseModel): - ask: str - answer: str - - rh = reversed(history) - ix = 0 - while ix < len(rh): - t = HisMsg(**rh[ix]) - print(t) - # 如果 t是ask, continue - pass + total_length += length_delta + summary.insert(0, c.dict()) data = json.dumps(summary) return data From 05532426c08a250ec4d7661fbecf79bd918b1ada Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 17:02:11 +0800 Subject: [PATCH 320/378] feat: truncated history --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index f309b532e..d83611af1 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -107,7 +107,7 @@ class BrainMemory(pydantic.BaseModel): def add_history(self, msg: Message): if msg.id: - if self.to_int(msg.id, 0) < self.to_int(self.last_history_id, -1): + if self.to_int(msg.id, 0) <= self.to_int(self.last_history_id, -1): return self.history.append(msg.dict()) self.is_dirty = True From 92402bedd4e2fe171e9ee9732b9ad120075e0da5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 17:09:12 +0800 Subject: [PATCH 321/378] feat: truncated history --- metagpt/memory/brain_memory.py | 1 + 1 file changed, 1 insertion(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index d83611af1..04ae6593a 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -110,6 +110,7 @@ class BrainMemory(pydantic.BaseModel): if self.to_int(msg.id, 0) <= self.to_int(self.last_history_id, -1): return self.history.append(msg.dict()) + self.last_history_id = str(msg.id) self.is_dirty = True def exists(self, text) -> bool: From 4c82298e8864f9e8f3712aa9bb6333079a015749 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 18:21:10 +0800 Subject: [PATCH 322/378] feat: truncated history --- metagpt/memory/brain_memory.py | 62 ++++++++++++++++++++++++----- metagpt/provider/metagpt_llm_api.py | 41 ++----------------- metagpt/roles/assistant.py | 2 +- 3 files changed, 56 insertions(+), 49 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 04ae6593a..e8a98c55b 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -15,6 +15,7 @@ import pydantic from metagpt import Message from metagpt.logs import logger +from metagpt.schema import RawMessage from metagpt.utils.redis import Redis @@ -54,17 +55,21 @@ class BrainMemory(pydantic.BaseModel): def history_text(self): if len(self.history) == 0 and not self.historical_summary: return "" - texts = [self.historical_summary] if self.historical_summary else [] - for m in self.history[:-1]: - if isinstance(m, Dict): - t = Message(**m).content - elif isinstance(m, Message): - t = m.content - else: - continue - texts.append(t) + try: + self.loads_raw_messages() + return self.dumps_raw_messages() + except: + texts = [self.historical_summary] if self.historical_summary else [] + for m in self.history[:-1]: + if isinstance(m, Dict): + t = Message(**m).content + elif isinstance(m, Message): + t = m.content + else: + continue + texts.append(t) - return "\n".join(texts) + return "\n".join(texts) @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": @@ -130,3 +135,40 @@ class BrainMemory(pydantic.BaseModel): v = self.last_talk self.last_talk = None return v + + def loads_raw_messages(self): + if not self.historical_summary: + return + vv = json.loads(self.historical_summary) + msgs = [] + for v in vv: + tag = set([MessageType.Talk.value]) if v.get("role") == "user" else set([MessageType.Answer.value]) + m = Message(content=v.get("content"), tags=tag) + msgs.append(m) + msgs.extend(self.history) + self.history = msgs + self.is_dirty = True + + def dumps_raw_messages(self, max_length: int = 0) -> str: + summary = [] + + total_length = 0 + for m in reversed(self.history): + msg = Message(**m) + c = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) + length_delta = len(msg.content) + if max_length > 0: + if total_length + length_delta > max_length: + left = max_length - total_length + if left > 0: + c.content = msg.content[0:left] + summary.insert(0, c) + break + + total_length += length_delta + summary.insert(0, c) + + self.historical_summary = json.dumps(summary) + self.history = [] + self.is_dirty = True + return self.historical_summary diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index d8d06aeaa..3ae65a623 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -5,35 +5,18 @@ @File : metagpt_llm_api.py @Desc : MetaGPT LLM related APIs """ -import json -from typing import Dict, List -from pydantic import BaseModel - -from metagpt.memory.brain_memory import MessageType +from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI -class HisMsg(BaseModel): - content: str - tags: set - id: str - - -class Conversion(BaseModel): - """See: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_format_inputs_to_ChatGPT_models.ipynb""" - - role: str - content: str - - class MetaGPTLLMAPI(OpenAIGPTAPI): """MetaGPT LLM api""" def __init__(self): super().__init__() - async def get_summary(self, history: List[Dict], max_words=200, keep_language: bool = False, **kwargs) -> str: + async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs) -> str: """ Return string in the following format： [ @@ -43,22 +26,4 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): {"role": "user", "content": "Orange."}, ] """ - summary = [] - - total_length = 0 - for m in reversed(history): - msg = HisMsg(**m) - c = Conversion(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) - length_delta = len(msg.content) - if total_length + length_delta > max_words: - left = max_words - total_length - if left > 0: - c.content = msg.content[0:left] - summary.insert(0, c.dict()) - break - - total_length += length_delta - summary.insert(0, c.dict()) - - data = json.dumps(summary) - return data + return memory.dumps_raw_messages(max_length=max_words) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 2f9059210..2fcb6f584 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -122,7 +122,7 @@ class Assistant(Role): if history_text == "": return last_talk history_summary = await self._llm.get_summary( - text=history_text, max_words=800, keep_language=True, history=self.memory.history + text=history_text, max_words=800, keep_language=True, memory=self.memory ) await self.memory.set_history_summary( history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS From 530d2f5b308a9c280853a20f51c2fac929c95134 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:03:41 +0800 Subject: [PATCH 323/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 113 +++++++++++++++++++++++++++++++++ metagpt/provider/openai_api.py | 110 -------------------------------- metagpt/roles/assistant.py | 20 +++--- 3 files changed, 123 insertions(+), 120 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e8a98c55b..7eda9c601 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -8,12 +8,16 @@ @Modified By: mashenquan, 2023/9/4. + redis memory cache. """ import json +import re from enum import Enum from typing import Dict, List +import openai import pydantic from metagpt import Message +from metagpt.config import CONFIG +from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.schema import RawMessage from metagpt.utils.redis import Redis @@ -36,6 +40,7 @@ class BrainMemory(pydantic.BaseModel): last_history_id: str = "" is_dirty: bool = False last_talk: str = None + llm_type: str def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -172,3 +177,111 @@ class BrainMemory(pydantic.BaseModel): self.history = [] self.is_dirty = True return self.historical_summary + + async def get_summary(self, text: str, llm, max_words=200, keep_language: bool = False, **kwargs): + max_token_count = DEFAULT_MAX_TOKENS + max_count = 100 + text_length = len(text) + while max_count > 0: + if text_length < max_token_count: + return await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language) + + padding_size = 20 if max_token_count > 20 else 0 + text_windows = self.split_texts(text, window_size=max_token_count - padding_size) + part_max_words = min(int(max_words / len(text_windows)) + 1, 100) + summaries = [] + for ws in text_windows: + response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) + summaries.append(response) + if len(summaries) == 1: + return summaries[0] + + # Merged and retry + text = "\n".join(summaries) + text_length = len(text) + + max_count -= 1 # safeguard + raise openai.error.InvalidRequestError("text too long") + + async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): + """Generate text summary""" + if len(text) < max_words: + return text + if keep_language: + command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly." + else: + command = f"Translate the above content into a summary of less than {max_words} words." + msg = text + "\n\n" + command + logger.debug(f"summary ask:{msg}") + response = await llm.aask(msg=msg, system_msgs=[]) + logger.debug(f"summary rsp: {response}") + return response + + async def get_title(self, text: str, llm, max_words=5, **kwargs) -> str: + """Generate text title""" + summary = await self.get_summary(text, max_words=500) + + language = CONFIG.language or DEFAULT_LANGUAGE + command = f"Translate the above summary into a {language} title of less than {max_words} words." + summaries = [summary, command] + msg = "\n".join(summaries) + logger.debug(f"title ask:{msg}") + response = await llm.aask(msg=msg, system_msgs=[]) + logger.debug(f"title rsp: {response}") + return response + + async def is_related(self, text1, text2, llm): + # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." + command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." + rsp = await llm.aask(msg=command, system_msgs=[]) + result = True if "TRUE" in rsp else False + p2 = text2.replace("\n", "") + p1 = text1.replace("\n", "") + logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n") + return result + + async def rewrite(self, sentence: str, context: str, llm): + # command = ( + # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" + # ) + command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" + rsp = await llm.aask(msg=command, system_msgs=[]) + logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n") + return rsp + + @staticmethod + def split_texts(text: str, window_size) -> List[str]: + """Splitting long text into sliding windows text""" + if window_size <= 0: + window_size = BrainMemory.DEFAULT_TOKEN_SIZE + total_len = len(text) + if total_len <= window_size: + return [text] + + padding_size = 20 if window_size > 20 else 0 + windows = [] + idx = 0 + data_len = window_size - padding_size + while idx < total_len: + if window_size + idx > total_len: # 不足一个滑窗 + windows.append(text[idx:]) + break + # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] + # window_size=3, padding_size=1： + # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... + # idx=2, | idx=5 | idx=8 | ... + w = text[idx : idx + window_size] + windows.append(w) + idx += data_len + + return windows + + @staticmethod + def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): + match = re.match(pattern, input_string) + if match: + return match.group(1), match.group(2) + else: + return None, input_string + + DEFAULT_TOKEN_SIZE = 500 diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 64267975e..231b568c7 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -8,10 +8,8 @@ """ import asyncio import random -import re import time import traceback -from typing import List import openai from openai.error import APIConnectionError @@ -24,7 +22,6 @@ from tenacity import ( ) from metagpt.config import CONFIG -from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS from metagpt.logs import logger from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.cost_manager import Costs @@ -223,112 +220,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs): - max_token_count = DEFAULT_MAX_TOKENS - max_count = 100 - text_length = len(text) - while max_count > 0: - if text_length < max_token_count: - return await self._get_summary(text=text, max_words=max_words, keep_language=keep_language) - - padding_size = 20 if max_token_count > 20 else 0 - text_windows = self.split_texts(text, window_size=max_token_count - padding_size) - part_max_words = min(int(max_words / len(text_windows)) + 1, 100) - summaries = [] - for ws in text_windows: - response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) - summaries.append(response) - if len(summaries) == 1: - return summaries[0] - - # Merged and retry - text = "\n".join(summaries) - text_length = len(text) - - max_count -= 1 # safeguard - raise openai.error.InvalidRequestError("text too long") - - async def _get_summary(self, text: str, max_words=20, keep_language: bool = False): - """Generate text summary""" - if len(text) < max_words: - return text - if keep_language: - command = f".Translate the above content into a summary of less than {max_words} words in language of the content strictly." - else: - command = f"Translate the above content into a summary of less than {max_words} words." - msg = text + "\n\n" + command - logger.debug(f"summary ask:{msg}") - response = await self.aask(msg=msg, system_msgs=[]) - logger.debug(f"summary rsp: {response}") - return response - - async def get_context_title(self, text: str, max_words=5, **kwargs) -> str: - """Generate text title""" - summary = await self.get_summary(text, max_words=500) - - language = CONFIG.language or DEFAULT_LANGUAGE - command = f"Translate the above summary into a {language} title of less than {max_words} words." - summaries = [summary, command] - msg = "\n".join(summaries) - logger.debug(f"title ask:{msg}") - response = await self.aask(msg=msg, system_msgs=[]) - logger.debug(f"title rsp: {response}") - return response - - async def is_related(self, text1, text2): - # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." - command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." - rsp = await self.aask(msg=command, system_msgs=[]) - result = True if "TRUE" in rsp else False - p2 = text2.replace("\n", "") - p1 = text1.replace("\n", "") - logger.info(f"IS_RELATED:\nParagraph 1: {p2}\nParagraph 2: {p1}\nRESULT: {result}\n") - return result - - async def rewrite(self, sentence: str, context: str): - # command = ( - # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" - # ) - command = f"{context}\n\nExtract relevant information from every preceding sentence and use it to succinctly supplement or rewrite the following text in brief and clear:\n{sentence}" - rsp = await self.aask(msg=command, system_msgs=[]) - logger.info(f"REWRITE:\nCommand: {command}\nRESULT: {rsp}\n") - return rsp - - @staticmethod - def split_texts(text: str, window_size) -> List[str]: - """Splitting long text into sliding windows text""" - if window_size <= 0: - window_size = OpenAIGPTAPI.DEFAULT_TOKEN_SIZE - total_len = len(text) - if total_len <= window_size: - return [text] - - padding_size = 20 if window_size > 20 else 0 - windows = [] - idx = 0 - data_len = window_size - padding_size - while idx < total_len: - if window_size + idx > total_len: # 不足一个滑窗 - windows.append(text[idx:]) - break - # 每个窗口少算padding_size自然就可实现滑窗功能, 比如: [1, 2, 3, 4, 5, 6, 7, ....] - # window_size=3, padding_size=1： - # [1, 2, 3], [3, 4, 5], [5, 6, 7], .... - # idx=2, | idx=5 | idx=8 | ... - w = text[idx : idx + window_size] - windows.append(w) - idx += data_len - - return windows - - @staticmethod - def extract_info(input_string, pattern=r"\[([A-Z]+)\]:\s*(.+)"): - match = re.match(pattern, input_string) - if match: - return match.group(1), match.group(2) - else: - return None, input_string - @staticmethod async def async_retry_call(func, *args, **kwargs): for i in range(OpenAIGPTAPI.MAX_TRY): @@ -371,7 +262,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") MAX_TRY = 5 - DEFAULT_TOKEN_SIZE = 500 if __name__ == "__main__": diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 2fcb6f584..d5467cafb 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -121,23 +121,23 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self._llm.get_summary( - text=history_text, max_words=800, keep_language=True, memory=self.memory + history_summary = await self.memory.get_summary( + text=history_text, max_words=800, keep_language=True, llm=self._llm ) - await self.memory.set_history_summary( - history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS - ) - if last_talk and await self._llm.is_related(last_talk, history_summary): # Merge relevant content. - last_talk = await self._llm.rewrite(sentence=last_talk, context=history_text) + # await self.memory.set_history_summary( + # history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS + # ) + if last_talk and await self.memory.is_related( + text1=last_talk, text2=history_summary, llm=self._llm + ): # Merge relevant content. + last_talk = await self.memory.rewrite(sentence=last_talk, context=history_text, llm=self._llm) return last_talk return last_talk @staticmethod def extract_info(input_string): - from metagpt.provider.openai_api import OpenAIGPTAPI - - return OpenAIGPTAPI.extract_info(input_string) + return BrainMemory.extract_info(input_string) def get_memory(self) -> str: return self.memory.json() From 4c873a91584286ea8bfb37a635a37b82eb5b3b3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:13:23 +0800 Subject: [PATCH 324/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 14 +++++++++++--- metagpt/roles/assistant.py | 20 +++++--------------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 7eda9c601..fea3b2512 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -178,13 +178,16 @@ class BrainMemory(pydantic.BaseModel): self.is_dirty = True return self.historical_summary - async def get_summary(self, text: str, llm, max_words=200, keep_language: bool = False, **kwargs): + async def summerize(self, llm, max_words=200, keep_language: bool = False, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 + text = self.history_text text_length = len(text) + summary = "" while max_count > 0: if text_length < max_token_count: - return await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language) + summary = await self._get_summary(text=text, llm=llm, max_words=max_words, keep_language=keep_language) + break padding_size = 20 if max_token_count > 20 else 0 text_windows = self.split_texts(text, window_size=max_token_count - padding_size) @@ -194,13 +197,18 @@ class BrainMemory(pydantic.BaseModel): response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) summaries.append(response) if len(summaries) == 1: - return summaries[0] + summary = summaries[0] + break # Merged and retry text = "\n".join(summaries) text_length = len(text) max_count -= 1 # safeguard + if not summary: + await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) + return summary + raise openai.error.InvalidRequestError("text too long") async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index d5467cafb..26711486f 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -45,7 +45,7 @@ class Assistant(Role): name=name, profile=profile, goal=goal, constraints=constraints, desc=desc, *args, **kwargs ) brain_memory = CONFIG.BRAIN_MEMORY - self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory() + self.memory = BrainMemory(**brain_memory) if brain_memory else BrainMemory(llm_type=CONFIG.LLM_TYPE) skill_path = Path(CONFIG.SKILL_PATH) if CONFIG.SKILL_PATH else None self.skills = SkillLoader(skill_yaml_file_name=skill_path) @@ -83,7 +83,7 @@ class Assistant(Role): self.memory.add_talk(Message(content=text)) async def _plan(self, rsp: str, **kwargs) -> bool: - skill, text = Assistant.extract_info(input_string=rsp) + skill, text = BrainMemory.extract_info(input_string=rsp) handlers = { MessageType.Talk.value: self.talk_handler, MessageType.Skill.value: self.skill_handler, @@ -121,24 +121,14 @@ class Assistant(Role): return None if history_text == "": return last_talk - history_summary = await self.memory.get_summary( - text=history_text, max_words=800, keep_language=True, llm=self._llm - ) - # await self.memory.set_history_summary( - # history_summary=history_summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS - # ) - if last_talk and await self.memory.is_related( - text1=last_talk, text2=history_summary, llm=self._llm - ): # Merge relevant content. + history_summary = await self.memory.summerize(max_words=800, keep_language=True, llm=self._llm) + if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): + # Merge relevant content. last_talk = await self.memory.rewrite(sentence=last_talk, context=history_text, llm=self._llm) return last_talk return last_talk - @staticmethod - def extract_info(input_string): - return BrainMemory.extract_info(input_string) - def get_memory(self) -> str: return self.memory.json() From 44706ba1416805083caec6683787157ec8df38ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:23:09 +0800 Subject: [PATCH 325/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index fea3b2512..adb1f0114 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -80,7 +80,7 @@ class BrainMemory(pydantic.BaseModel): async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: - return BrainMemory() + return BrainMemory(llm_type=CONFIG.LLM_TYPE) v = await redis.get(key=redis_key) logger.debug(f"REDIS GET {redis_key} {v}") if v: @@ -88,9 +88,11 @@ class BrainMemory(pydantic.BaseModel): bm = BrainMemory(**data) bm.is_dirty = False return bm - return BrainMemory() + return BrainMemory(llm_type=CONFIG.LLM_TYPE) async def dumps(self, redis_key: str, timeout_sec: int = 30 * 60, redis_conf: Dict = None): + if not self.is_dirty: + return redis = Redis(conf=redis_conf) if not redis.is_valid() or not redis_key: return False From 948d1577e4a51f673768f8c16c51e378e435c732 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:30:35 +0800 Subject: [PATCH 326/378] refactor: brain memory --- metagpt/memory/memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index bf9f0541c..f9dd5c1a3 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -8,7 +8,6 @@ from collections import defaultdict from typing import Iterable, Type -from metagpt.actions import Action from metagpt.schema import Message @@ -17,6 +16,8 @@ class Memory: def __init__(self): """Initialize an empty storage list and an empty index dictionary""" + from metagpt.actions import Action + self.storage: list[Message] = [] self.index: dict[Type[Action], list[Message]] = defaultdict(list) From c66012d087b9c80b207a256238142e6daf8e4a39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:45:42 +0800 Subject: [PATCH 327/378] refactor: brain memory --- metagpt/memory/memory.py | 3 +-- metagpt/provider/metagpt_llm_api.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/metagpt/memory/memory.py b/metagpt/memory/memory.py index f9dd5c1a3..bf9f0541c 100644 --- a/metagpt/memory/memory.py +++ b/metagpt/memory/memory.py @@ -8,6 +8,7 @@ from collections import defaultdict from typing import Iterable, Type +from metagpt.actions import Action from metagpt.schema import Message @@ -16,8 +17,6 @@ class Memory: def __init__(self): """Initialize an empty storage list and an empty index dictionary""" - from metagpt.actions import Action - self.storage: list[Message] = [] self.index: dict[Type[Action], list[Message]] = defaultdict(list) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 3ae65a623..95514cf53 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -6,7 +6,6 @@ @Desc : MetaGPT LLM related APIs """ -from metagpt.memory.brain_memory import BrainMemory from metagpt.provider import OpenAIGPTAPI @@ -16,7 +15,7 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init__(self): super().__init__() - async def get_summary(self, memory: BrainMemory, max_words=200, keep_language: bool = False, **kwargs) -> str: + async def get_summary(self, memory, max_words=200, keep_language: bool = False, **kwargs) -> str: """ Return string in the following format： [ From 0c21aa810f64743ac3a484d53c005bf654cbf3bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 19:49:51 +0800 Subject: [PATCH 328/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index adb1f0114..596928a4c 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -10,7 +10,7 @@ import json import re from enum import Enum -from typing import Dict, List +from typing import Dict, List, Optional import openai import pydantic @@ -40,7 +40,7 @@ class BrainMemory(pydantic.BaseModel): last_history_id: str = "" is_dirty: bool = False last_talk: str = None - llm_type: str + llm_type: Optional[str] = None def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) From 415e6d5686a231201cd9c2a92c0cefdda12893ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:25:50 +0800 Subject: [PATCH 329/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 38 +++++++++++++++++----------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 596928a4c..4f99de3c7 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -56,25 +56,25 @@ class BrainMemory(pydantic.BaseModel): texts = [Message(**m).content for m in self.knowledge] return "\n".join(texts) - @property - def history_text(self): - if len(self.history) == 0 and not self.historical_summary: - return "" - try: - self.loads_raw_messages() - return self.dumps_raw_messages() - except: - texts = [self.historical_summary] if self.historical_summary else [] - for m in self.history[:-1]: - if isinstance(m, Dict): - t = Message(**m).content - elif isinstance(m, Message): - t = m.content - else: - continue - texts.append(t) - - return "\n".join(texts) + # @property + # def history_text(self): + # if len(self.history) == 0 and not self.historical_summary: + # return "" + # try: + # self.loads_raw_messages() + # return self.dumps_raw_messages() + # except: + # texts = [self.historical_summary] if self.historical_summary else [] + # for m in self.history[:-1]: + # if isinstance(m, Dict): + # t = Message(**m).content + # elif isinstance(m, Message): + # t = m.content + # else: + # continue + # texts.append(t) + # + # return "\n".join(texts) @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": From 7abb1a3b9368c704dbec747755e107a72cc138ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:29:59 +0800 Subject: [PATCH 330/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 4f99de3c7..805ef1b27 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -44,12 +44,12 @@ class BrainMemory(pydantic.BaseModel): def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) - self.history.append(msg.dict()) + self.add_history(msg) self.is_dirty = True def add_answer(self, msg: Message): msg.add_tag(MessageType.Answer.value) - self.history.append(msg.dict()) + self.add_history(msg) self.is_dirty = True def get_knowledge(self) -> str: From c36e1d6f1a85c7d1fb4ad124efcfe2d40917d7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:41:46 +0800 Subject: [PATCH 331/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 805ef1b27..45a7c0691 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -103,7 +103,7 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def to_redis_key(prefix: str, user_id: str, chat_id: str): - return f"{prefix}:{chat_id}:{user_id}" + return f"{prefix}:{user_id}:{chat_id}" async def set_history_summary(self, history_summary, redis_key, redis_conf): if self.historical_summary == history_summary: @@ -294,4 +294,9 @@ class BrainMemory(pydantic.BaseModel): else: return None, input_string + def set_llm_type(self, v): + if v: + self.llm_type = v + self.is_dirty = True + DEFAULT_TOKEN_SIZE = 500 From 2be79730a020e0c455810087eb2e771df9d59f11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 20:49:36 +0800 Subject: [PATCH 332/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 9 +++++++-- metagpt/roles/assistant.py | 7 +++---- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 45a7c0691..b06bf1036 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -240,7 +240,8 @@ class BrainMemory(pydantic.BaseModel): logger.debug(f"title rsp: {response}") return response - async def is_related(self, text1, text2, llm): + @staticmethod + async def is_related(text1, text2, llm): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await llm.aask(msg=command, system_msgs=[]) @@ -295,8 +296,12 @@ class BrainMemory(pydantic.BaseModel): return None, input_string def set_llm_type(self, v): - if v: + if v and v != self.llm_type: self.llm_type = v self.is_dirty = True + @property + def is_history_available(self): + return self.history or self.historical_summary + DEFAULT_TOKEN_SIZE = 500 diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 26711486f..54c1e2f43 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -115,16 +115,15 @@ class Assistant(Role): return True async def refine_memory(self) -> str: - history_text = self.memory.history_text last_talk = self.memory.pop_last_talk() if last_talk is None: # No user feedback, unsure if past conversation is finished. return None - if history_text == "": + if not self.memory.is_history_available: return last_talk history_summary = await self.memory.summerize(max_words=800, keep_language=True, llm=self._llm) - if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): + if last_talk and await BrainMemory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. - last_talk = await self.memory.rewrite(sentence=last_talk, context=history_text, llm=self._llm) + last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) return last_talk return last_talk From 0703c29030587cb0c0b6a57907c5112c8fe84d0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:21:03 +0800 Subject: [PATCH 333/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 103 ++++++++++++++------------------- 1 file changed, 44 insertions(+), 59 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index b06bf1036..a9677bd66 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -18,6 +18,7 @@ import pydantic from metagpt import Message from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE, DEFAULT_MAX_TOKENS +from metagpt.llm import LLMType from metagpt.logs import logger from metagpt.schema import RawMessage from metagpt.utils.redis import Redis @@ -56,26 +57,6 @@ class BrainMemory(pydantic.BaseModel): texts = [Message(**m).content for m in self.knowledge] return "\n".join(texts) - # @property - # def history_text(self): - # if len(self.history) == 0 and not self.historical_summary: - # return "" - # try: - # self.loads_raw_messages() - # return self.dumps_raw_messages() - # except: - # texts = [self.historical_summary] if self.historical_summary else [] - # for m in self.history[:-1]: - # if isinstance(m, Dict): - # t = Message(**m).content - # elif isinstance(m, Message): - # t = m.content - # else: - # continue - # texts.append(t) - # - # return "\n".join(texts) - @staticmethod async def loads(redis_key: str, redis_conf: Dict = None) -> "BrainMemory": redis = Redis(conf=redis_conf) @@ -143,47 +124,19 @@ class BrainMemory(pydantic.BaseModel): self.last_talk = None return v - def loads_raw_messages(self): - if not self.historical_summary: - return - vv = json.loads(self.historical_summary) - msgs = [] - for v in vv: - tag = set([MessageType.Talk.value]) if v.get("role") == "user" else set([MessageType.Answer.value]) - m = Message(content=v.get("content"), tags=tag) - msgs.append(m) - msgs.extend(self.history) - self.history = msgs - self.is_dirty = True + async def summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + if self.llm_type == LLMType.METAGPT.value: + return await self._metagpt_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) - def dumps_raw_messages(self, max_length: int = 0) -> str: - summary = [] + return await self._openai_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) - total_length = 0 - for m in reversed(self.history): - msg = Message(**m) - c = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) - length_delta = len(msg.content) - if max_length > 0: - if total_length + length_delta > max_length: - left = max_length - total_length - if left > 0: - c.content = msg.content[0:left] - summary.insert(0, c) - break - - total_length += length_delta - summary.insert(0, c) - - self.historical_summary = json.dumps(summary) - self.history = [] - self.is_dirty = True - return self.historical_summary - - async def summerize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 - text = self.history_text + texts = [self.historical_summary] + for m in self.history: + texts.append(m.content) + text = "\n".join(texts) text_length = len(text) summary = "" while max_count > 0: @@ -210,9 +163,41 @@ class BrainMemory(pydantic.BaseModel): if not summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary - raise openai.error.InvalidRequestError("text too long") + async def _metagpt_summarize(self, max_words=200, **kwargs): + if not self.history: + return "" + + total_length = 0 + msgs = [] + for m in reversed(self.history): + delta = len(m.content) + if total_length + delta > max_words: + left = max_words - total_length + if left == 0: + break + m.content = m.content[0:left] + msgs.append(m) + break + msgs.append(m) + total_length += delta + self.history = msgs + self.is_dirty = True + await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF) + self.is_dirty = False + + return BrainMemory.to_metagpt_history_format(self.history) + + @staticmethod + def to_metagpt_history_format(history) -> str: + mmsg = [] + for m in reversed(history): + msg = Message(**m) + r = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) + mmsg.append(r) + return json.dumps(mmsg) + async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): """Generate text summary""" if len(text) < max_words: @@ -302,6 +287,6 @@ class BrainMemory(pydantic.BaseModel): @property def is_history_available(self): - return self.history or self.historical_summary + return bool(self.history or self.historical_summary) DEFAULT_TOKEN_SIZE = 500 From 8e30dfd84a8e516fe7a6ad7d993a7883ec728b98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:21:38 +0800 Subject: [PATCH 334/378] refactor: brain memory --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 54c1e2f43..66daef403 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -120,7 +120,7 @@ class Assistant(Role): return None if not self.memory.is_history_available: return last_talk - history_summary = await self.memory.summerize(max_words=800, keep_language=True, llm=self._llm) + history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm) if last_talk and await BrainMemory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) From 12b2fcd4be85b2dd013fc040d046dda938c38b72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:30:15 +0800 Subject: [PATCH 335/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a9677bd66..3d713ddfb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -171,16 +171,17 @@ class BrainMemory(pydantic.BaseModel): total_length = 0 msgs = [] - for m in reversed(self.history): + for i in reversed(self.history): + m = Message(**i) delta = len(m.content) if total_length + delta > max_words: left = max_words - total_length if left == 0: break m.content = m.content[0:left] - msgs.append(m) + msgs.append(m.dict()) break - msgs.append(m) + msgs.append(m.dict()) total_length += delta self.history = msgs self.is_dirty = True @@ -198,7 +199,8 @@ class BrainMemory(pydantic.BaseModel): mmsg.append(r) return json.dumps(mmsg) - async def _get_summary(self, text: str, llm, max_words=20, keep_language: bool = False): + @staticmethod + async def _get_summary(text: str, llm, max_words=20, keep_language: bool = False): """Generate text summary""" if len(text) < max_words: return text @@ -214,7 +216,7 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, text: str, llm, max_words=5, **kwargs) -> str: """Generate text title""" - summary = await self.get_summary(text, max_words=500) + summary = await self.summarize(text, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 24a3e725726338ed3b5a611489ce1af481692e2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:35:48 +0800 Subject: [PATCH 336/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 3d713ddfb..09a4915fc 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -181,7 +181,7 @@ class BrainMemory(pydantic.BaseModel): m.content = m.content[0:left] msgs.append(m.dict()) break - msgs.append(m.dict()) + msgs.append(i) total_length += delta self.history = msgs self.is_dirty = True From a4f36e0852f0804c327cc6cce016e00e28d0591c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:42:35 +0800 Subject: [PATCH 337/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 09a4915fc..e65459f1a 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -183,7 +183,7 @@ class BrainMemory(pydantic.BaseModel): break msgs.append(i) total_length += delta - self.history = msgs + self.history = msgs.reverse() self.is_dirty = True await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF) self.is_dirty = False From 5b3f6e0b6857210dacf115e171418d5893afdcf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 21:43:28 +0800 Subject: [PATCH 338/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e65459f1a..39e2ec43d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -183,7 +183,8 @@ class BrainMemory(pydantic.BaseModel): break msgs.append(i) total_length += delta - self.history = msgs.reverse() + msgs.reverse() + self.history = msgs self.is_dirty = True await self.dumps(redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS_CONF) self.is_dirty = False From 1df4121b12863793b23dcd7a11d6855b35eb752d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:06:54 +0800 Subject: [PATCH 339/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 39e2ec43d..2d191ccaa 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -194,7 +194,7 @@ class BrainMemory(pydantic.BaseModel): @staticmethod def to_metagpt_history_format(history) -> str: mmsg = [] - for m in reversed(history): + for m in history: msg = Message(**m) r = RawMessage(role="user" if MessageType.Talk.value in msg.tags else "assistant", content=msg.content) mmsg.append(r) From 1ce9ad54fd6dbe52f726b8977b18aae19049f23c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:24:20 +0800 Subject: [PATCH 340/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 2d191ccaa..e0e2ae1a0 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -228,8 +228,17 @@ class BrainMemory(pydantic.BaseModel): logger.debug(f"title rsp: {response}") return response + async def is_related(self, text1, text2, llm): + if self.llm_type == LLMType.METAGPT.value: + return await self._metagpt_is_related(text1=text1, text2=text2, llm=llm) + return await self._openai_is_related(text1=text1, text2=text2, llm=llm) + @staticmethod - async def is_related(text1, text2, llm): + async def _metagpt_is_related(**kwargs): + return False + + @staticmethod + async def _openai_is_related(text1, text2, llm, **kwargs): # command = f"{text1}\n{text2}\n\nIf the two sentences above are related, return [TRUE] brief and clear. Otherwise, return [FALSE]." command = f"{text2}\n\nIs there any sentence above related to the following sentence: {text1}.\nIf is there any relevance, return [TRUE] brief and clear. Otherwise, return [FALSE] brief and clear." rsp = await llm.aask(msg=command, system_msgs=[]) @@ -240,6 +249,14 @@ class BrainMemory(pydantic.BaseModel): return result async def rewrite(self, sentence: str, context: str, llm): + if self.llm_type == LLMType.METAGPT.value: + return await self._metagpt_rewrite(sentence=sentence, context=context, llm=llm) + return await self._openai_rewrite(sentence=sentence, context=context, llm=llm) + + async def _metagpt_rewrite(self, sentence: str, **kwargs): + return sentence + + async def _openai_rewrite(self, sentence: str, context: str, llm, **kwargs): # command = ( # f"{context}\n\nConsidering the content above, rewrite and return this sentence brief and clear:\n{sentence}" # ) From 7c6b0325d8a2001491d2ec25167ef638f417aa7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:27:49 +0800 Subject: [PATCH 341/378] refactor: brain memory --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 66daef403..397ddc94b 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -121,7 +121,7 @@ class Assistant(Role): if not self.memory.is_history_available: return last_talk history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm) - if last_talk and await BrainMemory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): + if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) return last_talk From f2da313548b07f81ce8e9299b2d96bb067ba7e4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 22:58:00 +0800 Subject: [PATCH 342/378] refactor: brain memory --- metagpt/actions/talk_action.py | 11 +++++++++++ metagpt/memory/brain_memory.py | 24 ++++++++++++++++++++++++ metagpt/provider/base_gpt_api.py | 19 +++++++++++++++---- 3 files changed, 50 insertions(+), 4 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 0e3762798..baef47eeb 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -6,10 +6,12 @@ @File : talk_action.py @Desc : Act as it’s a talk """ +import json from metagpt.actions import Action, ActionOutput from metagpt.config import CONFIG from metagpt.const import DEFAULT_LANGUAGE +from metagpt.llm import LLMType from metagpt.logs import logger @@ -63,6 +65,15 @@ class TalkAction(Action): return prompt async def run(self, *args, **kwargs) -> ActionOutput: + if CONFIG.LLM_TYPE == LLMType.METAGPT.value: + rsp = await self.llm.aask( + msg=self._talk, + knowledge_msgs=[{"knowledge": self._knowledge}] if self._knowledge else None, + history_msgs=json.loads(self._history_summary) if self._history_summary else None, + ) + self._rsp = ActionOutput(content=rsp) + return self._rsp + prompt = self.prompt rsp = await self.llm.aask(msg=prompt, system_msgs=[]) logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n") diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index e0e2ae1a0..0f9c1dbb6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -309,4 +309,28 @@ class BrainMemory(pydantic.BaseModel): def is_history_available(self): return bool(self.history or self.historical_summary) + @property + def history_text(self): + if self.llm_type == LLMType.METAGPT.value: + return self._get_metagpt_history_text() + return self._get_openai_history_text() + + def _get_metagpt_history_text(self): + return BrainMemory.to_metagpt_history_format(self.history) + + def _get_openai_history_text(self): + if len(self.history) == 0 and not self.historical_summary: + return "" + texts = [self.historical_summary] if self.historical_summary else [] + for m in self.history[:-1]: + if isinstance(m, Dict): + t = Message(**m).content + elif isinstance(m, Message): + t = m.content + else: + continue + texts.append(t) + + return "\n".join(texts) + DEFAULT_TOKEN_SIZE = 500 diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 7351e6916..f405ae902 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -38,11 +38,22 @@ class BaseGPTAPI(BaseChatbot): rsp = self.completion(message) return self.get_choice_text(rsp) - async def aask(self, msg: str, system_msgs: Optional[list[str]] = None, generator: bool = False) -> str: + async def aask( + self, + msg: str, + system_msgs: Optional[list[str]] = None, + history_msgs: Optional[list[dict[str, str]]] = None, + knowledge_msgs: Optional[list[dict[str, str]]] = None, + generator: bool = False, + ) -> str: + message = [] if system_msgs: - message = self._system_msgs(system_msgs) + [self._user_msg(msg)] - else: - message = [self._default_system_msg(), self._user_msg(msg)] + message = self._system_msgs(system_msgs) + if knowledge_msgs: + message.extend(knowledge_msgs) + if history_msgs: + message.extend(history_msgs) + message.append(self._user_msg(msg)) try: rsp = await self.acompletion_text(message, stream=True, generator=generator) except Exception as e: From f92aeb0e506e852d3551f7cf67b3574448e91712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:05:30 +0800 Subject: [PATCH 343/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 0f9c1dbb6..7677a9144 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -134,7 +134,8 @@ class BrainMemory(pydantic.BaseModel): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 texts = [self.historical_summary] - for m in self.history: + for i in self.history: + m = Message(**i) texts.append(m.content) text = "\n".join(texts) text_length = len(text) From 1b267d34dc986e8f18be63423783421d88e72eae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:08:46 +0800 Subject: [PATCH 344/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 7677a9144..f3a3e3563 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -161,7 +161,7 @@ class BrainMemory(pydantic.BaseModel): text_length = len(text) max_count -= 1 # safeguard - if not summary: + if summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary raise openai.error.InvalidRequestError("text too long") From 270b14e0360c41ede5fadaf71ea4b6e04b384dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:29:08 +0800 Subject: [PATCH 345/378] refactor: brain memory --- metagpt/actions/talk_action.py | 33 ++++++++++++++++++++++---------- metagpt/provider/base_gpt_api.py | 9 +++------ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index baef47eeb..2d473c7cf 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -64,22 +64,35 @@ class TalkAction(Action): logger.info(f"PROMPT: {prompt}") return prompt - async def run(self, *args, **kwargs) -> ActionOutput: - if CONFIG.LLM_TYPE == LLMType.METAGPT.value: - rsp = await self.llm.aask( - msg=self._talk, - knowledge_msgs=[{"knowledge": self._knowledge}] if self._knowledge else None, - history_msgs=json.loads(self._history_summary) if self._history_summary else None, - ) - self._rsp = ActionOutput(content=rsp) - return self._rsp - + async def run_old(self, *args, **kwargs) -> ActionOutput: prompt = self.prompt rsp = await self.llm.aask(msg=prompt, system_msgs=[]) logger.debug(f"PROMPT:{prompt}\nRESULT:{rsp}\n") self._rsp = ActionOutput(content=rsp) return self._rsp + async def run(self, *args, **kwargs) -> ActionOutput: + language = CONFIG.language or DEFAULT_LANGUAGE + system_msgs = [ + f"You are {CONFIG.agent_description}.", + "Your responses should align with the role-play agreement, " + "maintaining the character's persona and habits. When faced with unrelated questions, playfully " + "decline to answer without revealing your AI nature to preserve the character's image.", + "If the information is insufficient, you can search in the context or knowledge.", + f"Answer the following questions strictly in {language}, and the answers must follow the Markdown format.", + ] + format_msgs = [] + if self._knowledge: + format_msgs.append({"knowledge": self._knowledge}) + if self._history_summary: + if CONFIG.LLM_TYPE == LLMType.METAGPT.value: + format_msgs.append(json.loads(self._history_summary)) + else: + format_msgs.append({"context": self._history_summary}) + rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) + self._rsp = ActionOutput(content=rsp) + return self._rsp + __FORMATION__ = """Formation: "Capacity and role" defines the role you are currently playing; "[HISTORY_BEGIN]" and "[HISTORY_END]" tags enclose the historical conversation; "[KNOWLEDGE_BEGIN]" and "[KNOWLEDGE_END]" tags enclose the knowledge may help for your responses; diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index f405ae902..19f5fcfff 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -42,17 +42,14 @@ class BaseGPTAPI(BaseChatbot): self, msg: str, system_msgs: Optional[list[str]] = None, - history_msgs: Optional[list[dict[str, str]]] = None, - knowledge_msgs: Optional[list[dict[str, str]]] = None, + format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, ) -> str: message = [] if system_msgs: message = self._system_msgs(system_msgs) - if knowledge_msgs: - message.extend(knowledge_msgs) - if history_msgs: - message.extend(history_msgs) + if format_msgs: + message.extend(format_msgs) message.append(self._user_msg(msg)) try: rsp = await self.acompletion_text(message, stream=True, generator=generator) From b49c7f2d70e7b7f45d4e632c203b9fcecbfe52ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:38:04 +0800 Subject: [PATCH 346/378] refactor: brain memory --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 2d473c7cf..3c3db0841 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -88,7 +88,7 @@ class TalkAction(Action): if CONFIG.LLM_TYPE == LLMType.METAGPT.value: format_msgs.append(json.loads(self._history_summary)) else: - format_msgs.append({"context": self._history_summary}) + format_msgs.append({"knowledge": self._history_summary}) rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From d906bd1c81d534dbd1edee9d03e9e556a2805c99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:41:31 +0800 Subject: [PATCH 347/378] refactor: brain memory --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 3c3db0841..b5282c3e5 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -83,12 +83,12 @@ class TalkAction(Action): ] format_msgs = [] if self._knowledge: - format_msgs.append({"knowledge": self._knowledge}) + format_msgs.append({"role": "knowledge", "content": self._knowledge}) if self._history_summary: if CONFIG.LLM_TYPE == LLMType.METAGPT.value: format_msgs.append(json.loads(self._history_summary)) else: - format_msgs.append({"knowledge": self._history_summary}) + format_msgs.append({"role": "context", "content": self._history_summary}) rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From b1f7aa396895723b121f184d2fff559a72eb52be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:44:01 +0800 Subject: [PATCH 348/378] refactor: brain memory --- metagpt/actions/talk_action.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index b5282c3e5..85d99db49 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -83,12 +83,12 @@ class TalkAction(Action): ] format_msgs = [] if self._knowledge: - format_msgs.append({"role": "knowledge", "content": self._knowledge}) + format_msgs.append({"role": "assistant", "content": self._knowledge}) if self._history_summary: if CONFIG.LLM_TYPE == LLMType.METAGPT.value: format_msgs.append(json.loads(self._history_summary)) else: - format_msgs.append({"role": "context", "content": self._history_summary}) + format_msgs.append({"role": "assistant", "content": self._history_summary}) rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From f4eea02866cc76d9e3ceb809c466451abae91af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:54:56 +0800 Subject: [PATCH 349/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index f3a3e3563..cdf3d7fbb 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -164,7 +164,7 @@ class BrainMemory(pydantic.BaseModel): if summary: await self.set_history_summary(history_summary=summary, redis_key=CONFIG.REDIS_KEY, redis_conf=CONFIG.REDIS) return summary - raise openai.error.InvalidRequestError("text too long") + raise openai.error.InvalidRequestError(message="text too long", param=None) async def _metagpt_summarize(self, max_words=200, **kwargs): if not self.history: From 20fb71b0a3a6f2abebea6d81edf73c0a59f26afb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Thu, 7 Sep 2023 23:58:30 +0800 Subject: [PATCH 350/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index cdf3d7fbb..3dfa050b3 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -216,9 +216,9 @@ class BrainMemory(pydantic.BaseModel): logger.debug(f"summary rsp: {response}") return response - async def get_title(self, text: str, llm, max_words=5, **kwargs) -> str: + async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" - summary = await self.summarize(text, max_words=500) + summary = await self.summarize(max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 42d0281fbbba5ccd8f5646c2e7303ba1d5aa6f65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 00:00:41 +0800 Subject: [PATCH 351/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 3dfa050b3..a995244a6 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -218,7 +218,7 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" - summary = await self.summarize(max_words=500) + summary = await self.summarize(llm=llm, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE command = f"Translate the above summary into a {language} title of less than {max_words} words." From 8b5d83956d7cbc852fbeeb6e4006bc8d0712088e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:05:44 +0800 Subject: [PATCH 352/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 10 +++++++--- metagpt/provider/openai_api.py | 16 +++++++++++++++- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index a995244a6..9878fa750 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -124,13 +124,15 @@ class BrainMemory(pydantic.BaseModel): self.last_talk = None return v - async def summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + async def summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs): if self.llm_type == LLMType.METAGPT.value: return await self._metagpt_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) - return await self._openai_summarize(llm=llm, max_words=max_words, keep_language=keep_language, **kwargs) + return await self._openai_summarize( + llm=llm, max_words=max_words, keep_language=keep_language, limit=limit, **kwargs + ) - async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, **kwargs): + async def _openai_summarize(self, llm, max_words=200, keep_language: bool = False, limit: int = -1, **kwargs): max_token_count = DEFAULT_MAX_TOKENS max_count = 100 texts = [self.historical_summary] @@ -139,6 +141,8 @@ class BrainMemory(pydantic.BaseModel): texts.append(m.content) text = "\n".join(texts) text_length = len(text) + if limit > 0 and text_length < limit: + return text summary = "" while max_count > 0: if text_length < max_token_count: diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 231b568c7..9dbbaf7e5 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -22,7 +22,9 @@ from tenacity import ( ) from metagpt.config import CONFIG +from metagpt.llm import LLMType from metagpt.logs import logger +from metagpt.memory.brain_memory import BrainMemory from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.cost_manager import Costs from metagpt.utils.token_counter import ( @@ -261,6 +263,19 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise e raise openai.error.OpenAIError("Exceeds the maximum retries") + async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str: + """ + Return string in the following format： + [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Knock knock."}, + {"role": "assistant", "content": "Who's there?"}, + {"role": "user", "content": "Orange."}, + ] + """ + memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) + return await memory.summarize(llm=self._llm, max_length=max_words, keep_language=keep_language) + MAX_TRY = 5 @@ -269,4 +284,3 @@ if __name__ == "__main__": as dfas sad lkf sdkl sakdfsdk sjd jsk sdl sk dd sd asd fa sdf sad dd - .gitlab-ci.yml & base_test.py """ - OpenAIGPTAPI.split_texts(txt, 30) From 827505fca9838f7df57970174ae018df911f258d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:07:46 +0800 Subject: [PATCH 353/378] refactor: brain memory --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 9dbbaf7e5..85dfe8436 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -274,7 +274,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): ] """ memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) - return await memory.summarize(llm=self._llm, max_length=max_words, keep_language=keep_language) + return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) MAX_TRY = 5 From 6942cc91619e35a626cbfc5b33f5e27f856ebc42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:16:07 +0800 Subject: [PATCH 354/378] refactor: brain memory --- metagpt/llm.py | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 4772d2e6e..67ae42d62 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -8,15 +8,15 @@ """ from enum import Enum +import openai + from metagpt.config import CONFIG -from metagpt.provider.anthropic_api import Claude2 as Claude -from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM -from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM class LLMType(Enum): OPENAI = "OpenAI" METAGPT = "MetaGPT" + CLAUDE = "Claude" UNKNOWN = "UNKNOWN" @classmethod @@ -27,20 +27,18 @@ class LLMType(Enum): return cls.UNKNOWN -DEFAULT_LLM = OpenAI_LLM() -DEFAULT_METAGPT_LLM = MetaGPT_LLM() -CLAUDE_LLM = Claude() - - -async def ai_func(prompt): - """使用LLM进行QA - QA with LLMs - """ - return await DEFAULT_LLM.aask(prompt) - - class LLMFactory: @staticmethod def new_llm() -> object: - llm = OpenAI_LLM() if CONFIG.LLM_TYPE == LLMType.OPENAI.value else MetaGPT_LLM() - return llm + from metagpt.provider.anthropic_api import Claude2 as Claude + from metagpt.provider.metagpt_llm_api import MetaGPTLLMAPI as MetaGPT_LLM + from metagpt.provider.openai_api import OpenAIGPTAPI as OpenAI_LLM + + if CONFIG.LLM_TYPE == LLMType.OPENAI.value: + return OpenAI_LLM() + if CONFIG.LLM_TYPE == LLMType.METAGPT.value: + return MetaGPT_LLM() + if CONFIG.LLM_TYPE == LLMType.CLAUDE.value: + return Claude() + + raise openai.InvalidRequestError(message=f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}") From 525ca29c89d7279f082f0e7d237a6445dbdd61df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:17:52 +0800 Subject: [PATCH 355/378] refactor: brain memory --- metagpt/provider/openai_api.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 85dfe8436..de640aed7 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -24,7 +24,6 @@ from tenacity import ( from metagpt.config import CONFIG from metagpt.llm import LLMType from metagpt.logs import logger -from metagpt.memory.brain_memory import BrainMemory from metagpt.provider.base_gpt_api import BaseGPTAPI from metagpt.utils.cost_manager import Costs from metagpt.utils.token_counter import ( @@ -273,6 +272,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): {"role": "user", "content": "Orange."}, ] """ + from metagpt.memory.brain_memory import BrainMemory + memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) From 1254f93467ca8cd9cad34e3c6791ce9ffef3d633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:22:31 +0800 Subject: [PATCH 356/378] refactor: brain memory --- metagpt/memory/brain_memory.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 9878fa750..b8f9a2a15 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -42,6 +42,7 @@ class BrainMemory(pydantic.BaseModel): is_dirty: bool = False last_talk: str = None llm_type: Optional[str] = None + cacheable: bool = True def add_talk(self, msg: Message): msg.add_tag(MessageType.Talk.value) @@ -78,8 +79,9 @@ class BrainMemory(pydantic.BaseModel): if not redis.is_valid() or not redis_key: return False v = self.json() - await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) - logger.debug(f"REDIS SET {redis_key} {v}") + if self.cacheable: + await redis.set(key=redis_key, data=v, timeout_sec=timeout_sec) + logger.debug(f"REDIS SET {redis_key} {v}") self.is_dirty = False @staticmethod From 348cafa0b86096f96b4cb41fef197f04b5814256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 10:24:08 +0800 Subject: [PATCH 357/378] refactor: brain memory --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index de640aed7..514671488 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -274,7 +274,7 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ from metagpt.memory.brain_memory import BrainMemory - memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text) + memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) MAX_TRY = 5 From bed3d8c841dcd1c6901e9ecfcac5e855b4413164 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 11:54:26 +0800 Subject: [PATCH 358/378] refactor: brain memory --- metagpt/actions/talk_action.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index 85d99db49..f9ff76015 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -71,7 +71,8 @@ class TalkAction(Action): self._rsp = ActionOutput(content=rsp) return self._rsp - async def run(self, *args, **kwargs) -> ActionOutput: + @property + def aask_args(self): language = CONFIG.language or DEFAULT_LANGUAGE system_msgs = [ f"You are {CONFIG.agent_description}.", @@ -89,7 +90,11 @@ class TalkAction(Action): format_msgs.append(json.loads(self._history_summary)) else: format_msgs.append({"role": "assistant", "content": self._history_summary}) - rsp = await self.llm.aask(msg=self._talk, format_msgs=format_msgs, system_msgs=system_msgs) + return self._talk, format_msgs, system_msgs + + async def run(self, *args, **kwargs) -> ActionOutput: + msg, format_msgs, system_msgs = self.aask_args + rsp = await self.llm.aask(msg=msg, format_msgs=format_msgs, system_msgs=system_msgs) self._rsp = ActionOutput(content=rsp) return self._rsp From 5f3931820ec17d2de2aeef77b4294bfd3dc67b9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 12:08:05 +0800 Subject: [PATCH 359/378] refactor: brain memory --- metagpt/actions/talk_action.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/actions/talk_action.py b/metagpt/actions/talk_action.py index f9ff76015..eb619cb7e 100644 --- a/metagpt/actions/talk_action.py +++ b/metagpt/actions/talk_action.py @@ -87,7 +87,7 @@ class TalkAction(Action): format_msgs.append({"role": "assistant", "content": self._knowledge}) if self._history_summary: if CONFIG.LLM_TYPE == LLMType.METAGPT.value: - format_msgs.append(json.loads(self._history_summary)) + format_msgs.extend(json.loads(self._history_summary)) else: format_msgs.append({"role": "assistant", "content": self._history_summary}) return self._talk, format_msgs, system_msgs From 5903b3efbc33f3ec5ba68953980dac4c5c83dd3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:03:05 +0800 Subject: [PATCH 360/378] refactor: brain memory --- metagpt/llm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/llm.py b/metagpt/llm.py index 67ae42d62..eeb665872 100644 --- a/metagpt/llm.py +++ b/metagpt/llm.py @@ -41,4 +41,4 @@ class LLMFactory: if CONFIG.LLM_TYPE == LLMType.CLAUDE.value: return Claude() - raise openai.InvalidRequestError(message=f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}") + raise openai.InvalidRequestError(message=f"Unsupported LLM TYPE: {CONFIG.LLM_TYPE}", param=None) From ce6619a10c5aac43a715cfb53a6844c3c732e7d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:07:21 +0800 Subject: [PATCH 361/378] refactor: brain memory --- metagpt/provider/base_gpt_api.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 19f5fcfff..59da67d5b 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -48,6 +48,8 @@ class BaseGPTAPI(BaseChatbot): message = [] if system_msgs: message = self._system_msgs(system_msgs) + else: + message = [self._default_system_msg()] if format_msgs: message.extend(format_msgs) message.append(self._user_msg(msg)) From 1b71081c745f469a5f4529c30558f565a59bbe8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:08:29 +0800 Subject: [PATCH 362/378] refactor: brain memory --- metagpt/provider/base_gpt_api.py | 1 - 1 file changed, 1 deletion(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 59da67d5b..1b1187b72 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -45,7 +45,6 @@ class BaseGPTAPI(BaseChatbot): format_msgs: Optional[list[dict[str, str]]] = None, generator: bool = False, ) -> str: - message = [] if system_msgs: message = self._system_msgs(system_msgs) else: From 2c3ab2fae4be572d62e7fd5a54392e25993327b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 13:10:01 +0800 Subject: [PATCH 363/378] refactor: brain memory --- metagpt/provider/metagpt_llm_api.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/metagpt/provider/metagpt_llm_api.py b/metagpt/provider/metagpt_llm_api.py index 95514cf53..7e79f0ae5 100644 --- a/metagpt/provider/metagpt_llm_api.py +++ b/metagpt/provider/metagpt_llm_api.py @@ -14,15 +14,3 @@ class MetaGPTLLMAPI(OpenAIGPTAPI): def __init__(self): super().__init__() - - async def get_summary(self, memory, max_words=200, keep_language: bool = False, **kwargs) -> str: - """ - Return string in the following format： - [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Knock knock."}, - {"role": "assistant", "content": "Who's there?"}, - {"role": "user", "content": "Orange."}, - ] - """ - return memory.dumps_raw_messages(max_length=max_words) From dda55aec96ee25b5f44297b42b2075939b63f683 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 15:13:25 +0800 Subject: [PATCH 364/378] fixbug: llm missing --- metagpt/memory/brain_memory.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index b8f9a2a15..59d108a7d 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -156,7 +156,9 @@ class BrainMemory(pydantic.BaseModel): part_max_words = min(int(max_words / len(text_windows)) + 1, 100) summaries = [] for ws in text_windows: - response = await self._get_summary(text=ws, max_words=part_max_words, keep_language=keep_language) + response = await self._get_summary( + text=ws, llm=llm, max_words=part_max_words, keep_language=keep_language + ) summaries.append(response) if len(summaries) == 1: summary = summaries[0] From b58d2ff2d3ff64b4fd6a7c2279a6520a04e8e958 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 15:19:09 +0800 Subject: [PATCH 365/378] fixbug: llm missing --- metagpt/provider/openai_api.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 514671488..81be1975a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -263,15 +263,6 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): raise openai.error.OpenAIError("Exceeds the maximum retries") async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str: - """ - Return string in the following format： - [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Knock knock."}, - {"role": "assistant", "content": "Who's there?"}, - {"role": "user", "content": "Orange."}, - ] - """ from metagpt.memory.brain_memory import BrainMemory memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) From 95a5f1b9f1edc484b280b2b24277c9bf52926d58 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 8 Sep 2023 16:29:41 +0800 Subject: [PATCH 366/378] fixbug: context missing --- metagpt/roles/assistant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/roles/assistant.py b/metagpt/roles/assistant.py index 397ddc94b..84ca07c9a 100644 --- a/metagpt/roles/assistant.py +++ b/metagpt/roles/assistant.py @@ -123,7 +123,7 @@ class Assistant(Role): history_summary = await self.memory.summarize(max_words=800, keep_language=True, llm=self._llm) if last_talk and await self.memory.is_related(text1=last_talk, text2=history_summary, llm=self._llm): # Merge relevant content. - last_talk = await self.memory.rewrite(sentence=last_talk, llm=self._llm) + last_talk = await self.memory.rewrite(sentence=last_talk, context=history_summary, llm=self._llm) return last_talk return last_talk From 85dc0ad7d4522df3c2fc8bdb58c50f9029f25f33 Mon Sep 17 00:00:00 2001 From: shenchucheng Date: Sat, 9 Sep 2023 14:28:46 +0800 Subject: [PATCH 367/378] wait_exponential if RateLimitError --- metagpt/provider/base_gpt_api.py | 9 +--- metagpt/provider/openai_api.py | 71 ++++++-------------------------- 2 files changed, 13 insertions(+), 67 deletions(-) diff --git a/metagpt/provider/base_gpt_api.py b/metagpt/provider/base_gpt_api.py index 1b1187b72..e334e8a5d 100644 --- a/metagpt/provider/base_gpt_api.py +++ b/metagpt/provider/base_gpt_api.py @@ -9,7 +9,6 @@ from abc import abstractmethod from typing import Optional -from metagpt.logs import logger from metagpt.provider.base_chatbot import BaseChatbot @@ -52,13 +51,7 @@ class BaseGPTAPI(BaseChatbot): if format_msgs: message.extend(format_msgs) message.append(self._user_msg(msg)) - try: - rsp = await self.acompletion_text(message, stream=True, generator=generator) - except Exception as e: - logger.exception(f"{e}") - logger.info(f"ask:{msg}, error:{e}") - raise e - logger.info(f"ask:{msg}, anwser:{rsp}") + rsp = await self.acompletion_text(message, stream=True, generator=generator) return rsp def _extract_assistant_rsp(self, context): diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 81be1975a..7fc8b867a 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -7,17 +7,16 @@ Change cost control from global to company level. """ import asyncio -import random import time -import traceback import openai -from openai.error import APIConnectionError +from openai.error import APIConnectionError, RateLimitError from tenacity import ( after_log, retry, retry_if_exception_type, stop_after_attempt, + wait_exponential, wait_fixed, ) @@ -75,16 +74,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): """ def __init__(self): - self.llm = openai self.model = CONFIG.openai_api_model self.auto_max_tokens = False self.rpm = int(CONFIG.get("RPM", 10)) RateLimiter.__init__(self, rpm=self.rpm) async def _achat_completion_stream(self, messages: list[dict]) -> str: - response = await self.async_retry_call( - openai.ChatCompletion.acreate, **self._cons_kwargs(messages), stream=True - ) + response = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages), stream=True) # iterate through the stream of events async for chunk in response: chunk_message = chunk["choices"][0]["delta"] # extract the message @@ -118,12 +114,12 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return kwargs async def _achat_completion(self, messages: list[dict]) -> dict: - rsp = await self.async_retry_call(self.llm.ChatCompletion.acreate, **self._cons_kwargs(messages)) + rsp = await openai.ChatCompletion.acreate(**self._cons_kwargs(messages)) self._update_costs(rsp.get("usage")) return rsp def _chat_completion(self, messages: list[dict]) -> dict: - rsp = self.retry_call(self.llm.ChatCompletion.create, **self._cons_kwargs(messages)) + rsp = openai.ChatCompletion.create(**self._cons_kwargs(messages)) self._update_costs(rsp) return rsp @@ -144,6 +140,13 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): retry=retry_if_exception_type(APIConnectionError), retry_error_callback=log_and_reraise, ) + @retry( + stop=stop_after_attempt(6), + wait=wait_exponential(1), + after=after_log(logger, logger.level("WARNING").name), + retry=retry_if_exception_type(RateLimitError), + reraise=True, + ) async def acompletion_text(self, messages: list[dict], stream=False, generator: bool = False) -> str: """when streaming, print each token in place.""" if stream: @@ -221,58 +224,8 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): return CONFIG.max_tokens_rsp return get_max_completion_tokens(messages, self.model, CONFIG.max_tokens_rsp) - @staticmethod - async def async_retry_call(func, *args, **kwargs): - for i in range(OpenAIGPTAPI.MAX_TRY): - try: - rsp = await func(*args, **kwargs) - return rsp - except openai.error.RateLimitError as e: - random_time = random.uniform(0, 3) # 生成0到5秒之间的随机时间 - rounded_time = round(random_time, 1) # 保留一位小数，以实现0.1秒的精度 - logger.warning(f"Exception:{e}, sleeping for {rounded_time} seconds") - await asyncio.sleep(rounded_time) - continue - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e - raise openai.error.OpenAIError("Exceeds the maximum retries") - - @staticmethod - def retry_call(func, *args, **kwargs): - for i in range(OpenAIGPTAPI.MAX_TRY): - try: - rsp = func(*args, **kwargs) - return rsp - except openai.error.RateLimitError as e: - logger.warning(f"Exception:{e}") - continue - except ( - openai.error.AuthenticationError, - openai.error.PermissionError, - openai.error.InvalidAPIType, - openai.error.SignatureVerificationError, - ) as e: - logger.warning(f"Exception:{e}") - raise e - except Exception as e: - error_str = traceback.format_exc() - logger.error(f"Exception:{e}, stack:{error_str}") - raise e - raise openai.error.OpenAIError("Exceeds the maximum retries") - async def get_summary(self, text: str, max_words=200, keep_language: bool = False, **kwargs) -> str: from metagpt.memory.brain_memory import BrainMemory memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) - - MAX_TRY = 5 - - -if __name__ == "__main__": - txt = """ -as dfas sad lkf sdkl sakdfsdk sjd jsk sdl sk dd sd asd fa sdf sad dd -- .gitlab-ci.yml & base_test.py - """ From 19e78ff13e109b55aebb59ca2da2c9f02bcd78a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 9 Sep 2023 16:38:43 +0800 Subject: [PATCH 368/378] fixbug: get_title --- metagpt/memory/brain_memory.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 59d108a7d..78eeac758 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -226,6 +226,9 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" + if self.llm_type == LLMType.METAGPT.value: + return self.history[0] if self.history else "New" + summary = await self.summarize(llm=llm, max_words=500) language = CONFIG.language or DEFAULT_LANGUAGE From 1b6b24077e2f4b9fa37af2ee742a3e578c8efeee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 9 Sep 2023 16:43:42 +0800 Subject: [PATCH 369/378] fixbug: get_title --- metagpt/memory/brain_memory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/memory/brain_memory.py b/metagpt/memory/brain_memory.py index 78eeac758..be3736100 100644 --- a/metagpt/memory/brain_memory.py +++ b/metagpt/memory/brain_memory.py @@ -227,7 +227,7 @@ class BrainMemory(pydantic.BaseModel): async def get_title(self, llm, max_words=5, **kwargs) -> str: """Generate text title""" if self.llm_type == LLMType.METAGPT.value: - return self.history[0] if self.history else "New" + return Message(**self.history[0]).content if self.history else "New" summary = await self.summarize(llm=llm, max_words=500) From 768e934444bb0c2180240a9671eb61ce3218471d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Sep 2023 17:32:45 +0800 Subject: [PATCH 370/378] refactor: uuid --- metagpt/tools/iflytek_tts.py | 2 +- metagpt/utils/s3.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/metagpt/tools/iflytek_tts.py b/metagpt/tools/iflytek_tts.py index a91d8091b..cb87d2e7f 100644 --- a/metagpt/tools/iflytek_tts.py +++ b/metagpt/tools/iflytek_tts.py @@ -136,7 +136,7 @@ async def oas3_iflytek_tts(text: str, voice: str = "", app_id: str = "", api_key if not voice: voice = CONFIG.IFLYTEK_VOICE or DEFAULT_IFLYTEK_VOICE - filename = Path(__file__).parent / (str(uuid.uuid4()).replace("-", "") + ".mp3") + filename = Path(__file__).parent / (uuid.uuid4().hex + ".mp3") try: tts = IFlyTekTTS(app_id=app_id, api_key=api_key, api_secret=api_secret) await tts.synthesize_speech(text=text, output_file=str(filename), voice=voice) diff --git a/metagpt/utils/s3.py b/metagpt/utils/s3.py index 96b457972..dde68f720 100644 --- a/metagpt/utils/s3.py +++ b/metagpt/utils/s3.py @@ -132,7 +132,7 @@ class S3: async def cache(self, data: str, file_ext: str, format: str = "") -> str: """Save data to remote S3 and return url""" - object_name = str(uuid.uuid4()).replace("-", "") + file_ext + object_name = uuid.uuid4().hex + file_ext path = Path(__file__).parent pathname = path / object_name try: From 89be81524c963a64e5e21c4cc05126bf289eb63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Fri, 15 Sep 2023 21:56:39 +0800 Subject: [PATCH 371/378] feat: update skill specification --- .well-known/skills.yaml | 213 +++++++++++++++++++++++----------- metagpt/learn/skill_loader.py | 61 +++++++--- 2 files changed, 189 insertions(+), 85 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index d08d7aced..137bfcdb4 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -1,72 +1,149 @@ +skillapi: "0.1.0" + +info: + title: "Agent Skill Specification" + version: "1.0" + entities: Assistant: - skills: - - name: text_to_speech - description: Text-to-speech - id: text_to_speech.text_to_speech - x-prerequisite: - - name: AZURE_TTS_SUBSCRIPTION_KEY - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - - name: AZURE_TTS_REGION - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - arguments: - text: 'The text used for voice conversion. Required.' - lang: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States). The optional parameter are "English", "Chinese". Default value: "Chinese".' - voice: 'Default value: "zh-CN-XiaomoNeural".' - style: 'Speaking style to express different emotions like cheerfulness, empathy, and calm. The optional parameter values are "affectionate", "angry", "calm", "cheerful", "depressed", "disgruntled", "embarrassed", "envious", "fearful", "gentle", "sad", "serious". Default value: "affectionate".' - role: 'With roles, the same voice can act as a different age and gender. The optional parameter values are "Girl", "Boy", "OlderAdultFemale", "OlderAdultMale", "SeniorFemale", "SeniorMale", "YoungAdultFemale", "YoungAdultMale". Default value: "Girl".' - examples: - - ask: 'A girl says "hello world"' - answer: 'text_to_speech(text="hello world", role="Girl")' - - ask: 'A boy affectionate says "hello world"' - answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' - - ask: 'A boy says "你好"' - answer: 'text_to_speech(text="你好", role="Boy", lang="Chinese")' - - ask: 'How to speak "你好"?' - answer: 'text_to_speech(text="你好", lang="Chinese")' - returns: - type: string - format: base64 + summary: assistant + description: assistant + skills: + - name: text_to_speech + description: Text-to-speech + id: text_to_speech.text_to_speech + required: + oneOf: + - schema: + type: object + properties: + AZURE_TTS_SUBSCRIPTION_KEY: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + AZURE_TTS_REGION: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + - schema: + type: object + properties: + IFLYTEK_APP_ID: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_KEY: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_SECRET: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + parameters: + text: + description: 'The text used for voice conversion.' + required: true + type: string + lang: + description: 'The value can contain a language code such as en (English), or a locale such as en-US (English - United States).' + type: string + enum: + - English + - Chinese + default: Chinese + voice: + description: Name of voice styles + type: string + default: zh-CN-XiaomoNeural + style: + type: string + description: Speaking style to express different emotions like cheerfulness, empathy, and calm. + enum: + - affectionate + - angry + - calm + - cheerful + - depressed + - disgruntled + - embarrassed + - envious + - fearful + - gentle + - sad + - serious + default: affectionate + role: + type: string + description: With roles, the same voice can act as a different age and gender. + enum: + - Girl + - Boy + - OlderAdultFemale + - OlderAdultMale + - SeniorFemale + - SeniorMale + - YoungAdultFemale + - YoungAdultMale + default: Girl + examples: + - ask: 'A girl says "hello world"' + answer: 'text_to_speech(text="hello world", role="Girl")' + - ask: 'A boy affectionate says "hello world"' + answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' + - ask: 'A boy says "你好"' + answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + returns: + type: string + format: base64 - - name: text_to_image - description: Create a drawing based on the text. - id: text_to_image.text_to_image - x-prerequisite: - - name: OPENAI_API_KEY - description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" - - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL - description: "Model url." - arguments: - text: 'The text used for image conversion. Required.' - size_type: 'Default value: "512x512".' - examples: - - ask: 'Draw a girl' - answer: 'text_to_image(text="Draw a girl", size_type="512x512")' - - ask: 'Draw an apple' - answer: 'text_to_image(text="Draw an apple", size_type="512x512")' - - ask: 'Draw an apple picture' - answer: 'text_to_image(text="Draw an apple", size_type="512x512")' - - ask: 'Draw an apple image' - answer: 'text_to_image(text="Draw an apple", size_type="512x512")' - returns: - type: string - format: base64 + - name: text_to_image + description: Create a drawing based on the text. + id: text_to_image.text_to_image + required: + oneOf: + - name: OPENAI_API_KEY + type: string + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + type: string + description: "Model url." + parameters: + text: + description: 'The text used for image conversion.' + type: string + required: true + size_type: + description: size type + type: string + default: "512x512" + examples: + - ask: 'Draw a girl' + answer: 'text_to_image(text="Draw a girl", size_type="512x512")' + - ask: 'Draw an apple' + answer: 'text_to_image(text="Draw an apple", size_type="512x512")' + returns: + type: string + format: base64 - - name: web_search - description: Perform Google searches to provide real-time information. - id: web_search.web_search - x-prerequisite: - - name: SEARCH_ENGINE - description: "Supported values: serpapi/google/serper/ddg" - - name: SERPER_API_KEY - description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" - arguments: - query: 'The search query. Required.' - max_results: 'The number of search results to retrieve. Default value: 6.' - examples: - - ask: 'Search for information about artificial intelligence' - answer: 'web_search(query="Search for information about artificial intelligence", max_results=6)' - - ask: 'Find news articles about climate change' - answer: 'web_search(query="Find news articles about climate change", max_results=6)' - returns: - type: string \ No newline at end of file + - name: web_search + description: Perform Google searches to provide real-time information. + id: web_search.web_search + required: + - name: SEARCH_ENGINE + type: string + description: "Supported values: serpapi/google/serper/ddg" + - name: SERPER_API_KEY + type: string + description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + parameters: + query: + type: string + description: 'The search query.' + required: true + max_results: + type: number + default: 6 + description: 'The number of search results to retrieve.' + examples: + - ask: 'Search for information about artificial intelligence' + answer: 'web_search(query="Search for information about artificial intelligence", max_results=6)' + - ask: 'Find news articles about climate change' + answer: 'web_search(query="Find news articles about climate change", max_results=6)' + returns: + type: string diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index 83200bca6..b1d27db92 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -7,10 +7,10 @@ @Desc : Skill YAML Configuration Loader. """ from pathlib import Path -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Union import yaml -from pydantic import BaseModel, Field +from pydantic import BaseModel from metagpt.config import CONFIG @@ -25,29 +25,43 @@ class Returns(BaseModel): format: Optional[str] = None -class Prerequisite(BaseModel): - name: str - type: Optional[str] = None - description: Optional[str] = None - default: Optional[str] = None +class Parameter(BaseModel): + type: str + description: str = None class Skill(BaseModel): name: str - description: str - id: str - x_prerequisite: Optional[List[Prerequisite]] = Field(default=None, alias="x-prerequisite") - arguments: Dict + description: str = None + id: str = None + required: Optional[Union[List, Dict]] = None + parameters: Dict[str, Parameter] = None examples: List[Example] returns: Returns + @property + def arguments(self) -> Dict: + if not self.parameters: + return {} + ret = {} + for k, v in self.parameters.items(): + ret[k] = v.description if v.description else "" + return ret -class EntitySkills(BaseModel): + +class Entity(BaseModel): + name: str = None skills: List[Skill] +class Components(BaseModel): + pass + + class SkillsDeclaration(BaseModel): - entities: Dict[str, EntitySkills] + skillapi: str + entities: Dict[str, Entity] + components: Components = None class SkillLoader: @@ -60,8 +74,8 @@ class SkillLoader: def get_skill_list(self, entity_name: str = "Assistant") -> Dict: """Return the skill name based on the skill description.""" - entity_skills = self.get_entity(entity_name) - if not entity_skills: + entity = self.get_entity(entity_name) + if not entity: return {} agent_skills = CONFIG.agent_skills @@ -73,7 +87,7 @@ class SkillLoader: names = [AgentSkill(**i).name for i in agent_skills] description_to_name_mappings = {} - for s in entity_skills.skills: + for s in entity.skills: if s.name not in names: continue description_to_name_mappings[s.description] = s.name @@ -89,8 +103,21 @@ class SkillLoader: if sk.name == name: return sk - def get_entity(self, name) -> EntitySkills: + def get_entity(self, name) -> Entity: """Return a list of skills for the entity.""" if not self._skills: return None return self._skills.entities.get(name) + + +if __name__ == "__main__": + CONFIG.agent_skills = [ + {"id": 1, "name": "text_to_speech", "type": "builtin", "config": {}, "enabled": True}, + {"id": 2, "name": "text_to_image", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "ai_call", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "data_analysis", "type": "builtin", "config": {}, "enabled": True}, + {"id": 5, "name": "crawler", "type": "builtin", "config": {"engine": "ddg"}, "enabled": True}, + {"id": 6, "name": "knowledge", "type": "builtin", "config": {}, "enabled": True}, + ] + loader = SkillLoader() + print(loader.get_skill_list()) From 9fdf70658608d2a91d3648bf155d0ff4fa5b7d82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 16 Sep 2023 10:37:27 +0800 Subject: [PATCH 372/378] feat: +type --- .well-known/metagpt_oas3_api.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index 1e3cecb10..e21cc2d01 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -14,8 +14,10 @@ paths: /tts/azsure: x-prerequisite: - name: AZURE_TTS_SUBSCRIPTION_KEY + type: string description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - name: AZURE_TTS_REGION + type: string description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" post: summary: "Convert Text to Base64-encoded .wav File Stream" @@ -76,10 +78,13 @@ paths: /tts/iflytek: x-prerequisite: - name: IFLYTEK_APP_ID + type: string description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" - name: IFLYTEK_API_KEY + type: string description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" - name: IFLYTEK_API_SECRET + type: string description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" post: summary: "Convert Text to Base64-encoded .mp3 File Stream" @@ -133,6 +138,7 @@ paths: /txt2img/openai: x-prerequisite: - name: OPENAI_API_KEY + type: string description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: "Convert Text to Base64-encoded Image Data Stream" @@ -174,6 +180,7 @@ paths: /txt2embedding/openai: x-prerequisite: - name: OPENAI_API_KEY + type: string description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" post: summary: Text to embedding @@ -216,6 +223,7 @@ paths: /txt2image/metagpt: x-prerequisite: - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + type: string description: "Model url." post: summary: "Text to Image" From b4493052e7a3eb2533e5a642491a5e9c1c0e5e98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 16 Sep 2023 14:56:38 +0800 Subject: [PATCH 373/378] feat: +x-prerequisite --- .well-known/metagpt_oas3_api.yaml | 71 ++++++++++++++++--------- .well-known/skills.yaml | 86 ++++++++++++++++++------------- 2 files changed, 96 insertions(+), 61 deletions(-) diff --git a/.well-known/metagpt_oas3_api.yaml b/.well-known/metagpt_oas3_api.yaml index e21cc2d01..0a702e8b6 100644 --- a/.well-known/metagpt_oas3_api.yaml +++ b/.well-known/metagpt_oas3_api.yaml @@ -13,12 +13,17 @@ servers: paths: /tts/azsure: x-prerequisite: - - name: AZURE_TTS_SUBSCRIPTION_KEY - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - - name: AZURE_TTS_REGION - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + configurations: + AZURE_TTS_SUBSCRIPTION_KEY: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + AZURE_TTS_REGION: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + required: + allOf: + - AZURE_TTS_SUBSCRIPTION_KEY + - AZURE_TTS_REGION post: summary: "Convert Text to Base64-encoded .wav File Stream" description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" @@ -77,15 +82,21 @@ paths: /tts/iflytek: x-prerequisite: - - name: IFLYTEK_APP_ID - type: string - description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" - - name: IFLYTEK_API_KEY - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" - - name: IFLYTEK_API_SECRET - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + configurations: + IFLYTEK_APP_ID: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_KEY: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_SECRET: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + required: + allOf: + - IFLYTEK_APP_ID + - IFLYTEK_API_KEY + - IFLYTEK_API_SECRET post: summary: "Convert Text to Base64-encoded .mp3 File Stream" description: "For more details, check out: [iFlyTek](https://console.xfyun.cn/services/tts)" @@ -137,9 +148,13 @@ paths: /txt2img/openai: x-prerequisite: - - name: OPENAI_API_KEY - type: string - description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + configurations: + OPENAI_API_KEY: + type: string + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + required: + allOf: + - OPENAI_API_KEY post: summary: "Convert Text to Base64-encoded Image Data Stream" operationId: openai_text_to_image.oas3_openai_text_to_image @@ -179,9 +194,13 @@ paths: description: "Internal Server Error" /txt2embedding/openai: x-prerequisite: - - name: OPENAI_API_KEY - type: string - description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + configurations: + OPENAI_API_KEY: + type: string + description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" + required: + allOf: + - OPENAI_API_KEY post: summary: Text to embedding operationId: openai_text_to_embedding.oas3_openai_text_to_embedding @@ -222,9 +241,13 @@ paths: /txt2image/metagpt: x-prerequisite: - - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL - type: string - description: "Model url." + configurations: + METAGPT_TEXT_TO_IMAGE_MODEL_URL: + type: string + description: "Model url." + required: + allOf: + - METAGPT_TEXT_TO_IMAGE_MODEL_URL post: summary: "Text to Image" description: "Generate an image from the provided text using the MetaGPT Text-to-Image API." diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 137bfcdb4..05465454a 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -12,29 +12,32 @@ entities: - name: text_to_speech description: Text-to-speech id: text_to_speech.text_to_speech - required: - oneOf: - - schema: - type: object - properties: - AZURE_TTS_SUBSCRIPTION_KEY: - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - AZURE_TTS_REGION: - type: string - description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" - - schema: - type: object - properties: - IFLYTEK_APP_ID: - type: string - description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" - IFLYTEK_API_KEY: - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" - IFLYTEK_API_SECRET: - type: string - description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + x-prerequisite: + configurations: + AZURE_TTS_SUBSCRIPTION_KEY: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + AZURE_TTS_REGION: + type: string + description: "For more details, check out: [Azure Text-to_Speech](https://learn.microsoft.com/en-us/azure/ai-services/speech-service/language-support?tabs=tts)" + IFLYTEK_APP_ID: + type: string + description: "Application ID is used to access your iFlyTek service API, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_KEY: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + IFLYTEK_API_SECRET: + type: string + description: "WebAPI argument, see: `https://console.xfyun.cn/services/tts`" + required: + oneOf: + - allOf: + - AZURE_TTS_SUBSCRIPTION_KEY + - AZURE_TTS_REGION + - allOf: + - IFLYTEK_APP_ID + - IFLYTEK_API_KEY + - IFLYTEK_API_SECRET parameters: text: description: 'The text used for voice conversion.' @@ -51,9 +54,9 @@ entities: description: Name of voice styles type: string default: zh-CN-XiaomoNeural - style: + style: type: string - description: Speaking style to express different emotions like cheerfulness, empathy, and calm. + description: Speaking style to express different emotions like cheerfulness, empathy, and calm. enum: - affectionate - angry @@ -95,16 +98,20 @@ entities: - name: text_to_image description: Create a drawing based on the text. id: text_to_image.text_to_image - required: - oneOf: - - name: OPENAI_API_KEY + x-prerequisite: + configurations: + OPENAI_API_KEY: type: string description: "OpenAI API key, For more details, checkout: `https://platform.openai.com/account/api-keys`" - - name: METAGPT_TEXT_TO_IMAGE_MODEL_URL + METAGPT_TEXT_TO_IMAGE_MODEL_URL: type: string description: "Model url." + required: + oneOf: + - OPENAI_API_KEY + - METAGPT_TEXT_TO_IMAGE_MODEL_URL parameters: - text: + text: description: 'The text used for image conversion.' type: string required: true @@ -124,13 +131,18 @@ entities: - name: web_search description: Perform Google searches to provide real-time information. id: web_search.web_search - required: - - name: SEARCH_ENGINE - type: string - description: "Supported values: serpapi/google/serper/ddg" - - name: SERPER_API_KEY - type: string - description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + x-prerequisite: + configurations: + SEARCH_ENGINE: + type: string + description: "Supported values: serpapi/google/serper/ddg" + SERPER_API_KEY: + type: string + description: "SERPER API KEY, For more details, checkout: `https://serper.dev/api-key`" + required: + allOf: + - SEARCH_ENGINE + - SERPER_API_KEY parameters: query: type: string From ad71adb2091bbefb948cad48bc70c74891226bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Sat, 16 Sep 2023 15:02:24 +0800 Subject: [PATCH 374/378] feat: +x-prerequisite --- metagpt/learn/skill_loader.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/metagpt/learn/skill_loader.py b/metagpt/learn/skill_loader.py index b1d27db92..dff5e26ae 100644 --- a/metagpt/learn/skill_loader.py +++ b/metagpt/learn/skill_loader.py @@ -7,10 +7,10 @@ @Desc : Skill YAML Configuration Loader. """ from pathlib import Path -from typing import Dict, List, Optional, Union +from typing import Dict, List, Optional import yaml -from pydantic import BaseModel +from pydantic import BaseModel, Field from metagpt.config import CONFIG @@ -34,7 +34,7 @@ class Skill(BaseModel): name: str description: str = None id: str = None - required: Optional[Union[List, Dict]] = None + x_prerequisite: Dict = Field(default=None, alias="x-prerequisite") parameters: Dict[str, Parameter] = None examples: List[Example] returns: Returns From 4bf3510832e1114c9418b56d02f215c48334964f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Sep 2023 14:13:28 +0800 Subject: [PATCH 375/378] feat: +unit test --- tests/metagpt/learn/test_skill_loader.py | 41 ++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 tests/metagpt/learn/test_skill_loader.py diff --git a/tests/metagpt/learn/test_skill_loader.py b/tests/metagpt/learn/test_skill_loader.py new file mode 100644 index 000000000..5bc0e776f --- /dev/null +++ b/tests/metagpt/learn/test_skill_loader.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +""" +@Time : 2023/9/19 +@Author : mashenquan +@File : test_skill_loader.py +@Desc : Unit tests. +""" + +from metagpt.config import CONFIG +from metagpt.learn.skill_loader import SkillLoader + + +def test_suite(): + CONFIG.agent_skills = [ + {"id": 1, "name": "text_to_speech", "type": "builtin", "config": {}, "enabled": True}, + {"id": 2, "name": "text_to_image", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "ai_call", "type": "builtin", "config": {}, "enabled": True}, + {"id": 3, "name": "data_analysis", "type": "builtin", "config": {}, "enabled": True}, + {"id": 5, "name": "crawler", "type": "builtin", "config": {"engine": "ddg"}, "enabled": True}, + {"id": 6, "name": "knowledge", "type": "builtin", "config": {}, "enabled": True}, + {"id": 6, "name": "web_search", "type": "builtin", "config": {}, "enabled": True}, + ] + loader = SkillLoader() + skills = loader.get_skill_list() + assert skills + assert len(skills) >= 3 + for desc, name in skills.items(): + assert desc + assert name + + entity = loader.get_entity("Assistant") + assert entity + assert entity.skills + for sk in entity.skills: + assert sk + assert sk.arguments + + +if __name__ == "__main__": + test_suite() From c69928a1745a84bb9a25a040ac50a59a849807ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Tue, 19 Sep 2023 21:33:23 +0800 Subject: [PATCH 376/378] refactor: example --- .well-known/skills.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.well-known/skills.yaml b/.well-known/skills.yaml index 05465454a..c19a9501e 100644 --- a/.well-known/skills.yaml +++ b/.well-known/skills.yaml @@ -10,7 +10,7 @@ entities: description: assistant skills: - name: text_to_speech - description: Text-to-speech + description: Generate a voice file from the input text, text-to-speech id: text_to_speech.text_to_speech x-prerequisite: configurations: @@ -90,7 +90,7 @@ entities: - ask: 'A boy affectionate says "hello world"' answer: 'text_to_speech(text="hello world", role="Boy", style="affectionate")' - ask: 'A boy says "你好"' - answer: 'text_to_speech(text="hello world", role="Boy", lang="Chinese")' + answer: 'text_to_speech(text="你好", role="Boy", lang="Chinese")' returns: type: string format: base64 From 49f55ad3746da6c71587535e6fa9f85695bffb11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Sep 2023 11:37:33 +0800 Subject: [PATCH 377/378] feat: +LLM_TYPE: OpenAI --- config/config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/config.yaml b/config/config.yaml index 5c8dea03e..71744aa7f 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -11,6 +11,7 @@ OPENAI_API_BASE: "https://api.openai.com/v1" OPENAI_API_MODEL: "gpt-4" MAX_TOKENS: 1500 RPM: 10 +LLM_TYPE: OpenAI #### if Anthropic #Anthropic_API_KEY: "YOUR_API_KEY" From 56bf0b9b97c69e0aa0a49ddf35d576945e38d236 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=8E=98=E6=9D=83=20=E9=A9=AC?= Date: Wed, 20 Sep 2023 17:45:47 +0800 Subject: [PATCH 378/378] fixbug: max_words --- metagpt/provider/openai_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metagpt/provider/openai_api.py b/metagpt/provider/openai_api.py index 7fc8b867a..953043912 100644 --- a/metagpt/provider/openai_api.py +++ b/metagpt/provider/openai_api.py @@ -228,4 +228,4 @@ class OpenAIGPTAPI(BaseGPTAPI, RateLimiter): from metagpt.memory.brain_memory import BrainMemory memory = BrainMemory(llm_type=LLMType.OPENAI.value, historical_summary=text, cacheable=False) - return await memory.summarize(llm=self, max_length=max_words, keep_language=keep_language) + return await memory.summarize(llm=self, max_words=max_words, keep_language=keep_language)